openai api: max_tokens -> max_completion_tokens (#1222)

Author: shihaobai

lightllm/models/deepseek3_2/triton_kernel/act_quant.py

@@ -1,4 +1,6 @@
-# Adapted from https://github.com/sgl-project/sglang/blob/ce6b17c0f94e6bf53633c8f324176a891e67fa7f/python/sglang/srt/layers/attention/nsa/triton_kernel.py
+# Adapted from sglang Triton kernel implementation:
+# https://github.com/sgl-project/sglang/blob/ce6b17c0f94e6bf53633c8f324176a891e67fa7f/
+# python/sglang/srt/layers/attention/nsa/triton_kernel.py
 from typing import Optional, Tuple
 
 import torch
@@ -91,7 +93,8 @@ def act_quant(
     Quantizes the input tensor `x` using block-wise quantization with Triton.
 
     Args:
-        x (torch.Tensor): The input tensor to be quantized. Must be contiguous and its last dimension size must be divisible by `block_size`.
+        x (torch.Tensor): The input tensor to be quantized. Must be
+                          contiguous and its last dimension size must be divisible by `block_size`.
         block_size (int, optional): The size of the blocks to be used for quantization. Default is 128.
         scale_fmt (Optional[str], optional): The format of the scale. Default is None.
     Returns:

lightllm/server/api_models.py

@@ -1,4 +1,5 @@
 import time
+from typing_extensions import deprecated
 import uuid
 
 from pydantic import BaseModel, Field, field_validator, model_validator
@@ -119,7 +120,10 @@ class CompletionRequest(BaseModel):
     # prompt: string or tokens
     prompt: Union[str, List[str], List[int], List[List[int]]]
     suffix: Optional[str] = None
-    max_tokens: Optional[int] = 8192
+    max_tokens: Optional[int] = Field(
+        default=16384, deprecated="max_tokens is deprecated, please use max_completion_tokens instead"
+    )
+    max_completion_tokens: Optional[int] = None
     temperature: Optional[float] = 1.0
     top_p: Optional[float] = 1.0
     n: Optional[int] = 1
@@ -192,7 +196,10 @@ class ChatCompletionRequest(BaseModel):
     stream: Optional[bool] = False
     stream_options: Optional[StreamOptions] = None
     stop: Optional[Union[str, List[str]]] = None
-    max_tokens: Optional[int] = 8192
+    max_tokens: Optional[int] = Field(
+        default=16384, deprecated="max_tokens is deprecated, please use max_completion_tokens instead"
+    )
+    max_completion_tokens: Optional[int] = None
     presence_penalty: Optional[float] = 0.0
     frequency_penalty: Optional[float] = 0.0
     logit_bias: Optional[Dict[str, float]] = None

lightllm/server/api_openai.py

@@ -233,14 +233,19 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req
         "top_p": request.top_p,
         "top_k": request.top_k,
         "ignore_eos": request.ignore_eos,
-        "max_new_tokens": request.max_tokens,
-        "stop_sequences": request.stop,
         "n": request.n,
         "best_of": request.n,
         "add_special_tokens": False,
         "seed": request.seed,
     }
 
+    if request.max_completion_tokens is not None:
+        sampling_params_dict["max_new_tokens"] = request.max_completion_tokens
+    elif request.max_tokens is not None:
+        sampling_params_dict["max_new_tokens"] = request.max_tokens
+    if request.stop is not None:
+        sampling_params_dict["stop_sequences"] = request.stop
+
     # Structured output handling
     if request.response_format:
         if request.response_format.type == "json_schema":
@@ -571,13 +576,17 @@ async def completions_impl(request: CompletionRequest, raw_request: Request) ->
         "top_p": request.top_p,
         "top_k": request.top_k,
         "ignore_eos": request.ignore_eos,
-        "max_new_tokens": request.max_tokens,
-        "stop_sequences": request.stop,
         "n": request.n,
         "best_of": request.best_of,
         "add_special_tokens": False,
         "seed": request.seed,
     }
+    if request.max_completion_tokens is not None:
+        sampling_params_dict["max_new_tokens"] = request.max_completion_tokens
+    elif request.max_tokens is not None:
+        sampling_params_dict["max_new_tokens"] = request.max_tokens
+    if request.stop is not None:
+        sampling_params_dict["stop_sequences"] = request.stop
 
     if request.response_format:
         if request.response_format.type == "json_schema":

lightllm/server/core/objs/py_sampling_params.py

@@ -38,7 +38,7 @@ def __init__(
         top_k: int = None,  # -1 is for all
         ignore_eos: bool = False,
         image_max_patch_num: int = -1,
-        max_new_tokens: int = 16,
+        max_new_tokens: int = 16384,
         min_new_tokens: int = 1,
         stop_sequences: Optional[Union[str, List[str], List[List[int]]]] = None,  # 停止句子条件
         skip_special_tokens: bool = True,  # whether to skip special tokens when decoding
@@ -142,9 +142,9 @@ def verify(self):
         if self.top_k < -1 or self.top_k == 0:
             raise ValueError(f"top_k must be -1 (disable), or at least 1, got {self.top_k}.")
         if self.max_new_tokens < 1:
-            raise ValueError(f"max_new_tokens must be at least 1 , got {self.max_new_tokens}.")
+            raise ValueError(f"max_new_tokens must be at least 1, got {self.max_new_tokens}.")
         if self.min_new_tokens < 1:
-            raise ValueError(f"min_new_tokens must be at least 1 , got {self.min_new_tokens}.")
+            raise ValueError(f"min_new_tokens must be at least 1, got {self.min_new_tokens}.")
         if self.min_new_tokens > self.max_new_tokens:
             raise ValueError(
                 f"min_new_tokens must <= max_new_tokens, but got min {self.min_new_tokens}, max {self.max_new_tokens}."

lightllm/server/core/objs/sampling_params.py

@@ -345,7 +345,7 @@ def init(self, tokenizer, **kwargs):
         self.top_k = kwargs.get("top_k", SamplingParams._top_k)
         self.ignore_eos = kwargs.get("ignore_eos", False)
         self.image_max_patch_num = kwargs.get("image_max_patch_num", -1)
-        self.max_new_tokens = kwargs.get("max_new_tokens", 16)
+        self.max_new_tokens = kwargs.get("max_new_tokens", 16384)
         self.min_new_tokens = kwargs.get("min_new_tokens", 1)
         self.input_penalty = kwargs.get("input_penalty", DEFAULT_INPUT_PENALTY)
         self.group_request_id = kwargs.get("group_request_id", -1)
@@ -440,14 +440,13 @@ def verify(self):
         if self.top_k < -1 or self.top_k == 0:
             raise ValueError(f"top_k must be -1 (disable), or at least 1, got {self.top_k}.")
         if self.max_new_tokens < 1:
-            raise ValueError(f"max_new_tokens must be at least 1, got {self.max_new_tokens}.")
+            raise ValueError(f"max_new_tokens must be at least 1 , got {self.max_new_tokens}.")
         if self.min_new_tokens < 1:
-            raise ValueError(f"min_new_tokens must be at least 1, got {self.min_new_tokens}.")
+            raise ValueError(f"min_new_tokens must be at least 1 , got {self.min_new_tokens}.")
         if self.min_new_tokens > self.max_new_tokens:
             raise ValueError(
                 f"min_new_tokens must <= max_new_tokens, but got min {self.min_new_tokens}, max {self.max_new_tokens}."
             )
-
         self._verify_allowed_token_ids()
         self._verify_grammar_constraint()