From b75f6799d55fd53c4e6b65dc98ebbbe0621e683d Mon Sep 17 00:00:00 2001 From: root Date: Tue, 31 Mar 2026 12:03:33 +0800 Subject: [PATCH 1/6] fix decode token --- fastdeploy/engine/common_engine.py | 5 ++ tests/engine/test_decode_token.py | 106 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 tests/engine/test_decode_token.py diff --git a/fastdeploy/engine/common_engine.py b/fastdeploy/engine/common_engine.py index 28776b53ede..989f4681e57 100644 --- a/fastdeploy/engine/common_engine.py +++ b/fastdeploy/engine/common_engine.py @@ -1726,6 +1726,11 @@ def _decode_token(self, token_ids, req_id, is_end): token_ids = cum_tokens[prefix_offset:read_offset] else: token_ids = [] + + if is_end and delta_text == "" and len(cum_tokens) > 0: + read_offset = self.data_processor.decode_status[req_id][1] + token_ids = cum_tokens[read_offset:] + if is_end: del self.data_processor.decode_status[req_id] return delta_text, token_ids diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py new file mode 100644 index 00000000000..a9f64a0727e --- /dev/null +++ b/tests/engine/test_decode_token.py @@ -0,0 +1,106 @@ +""" +# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +from fastdeploy.engine.common_engine import EngineService +from fastdeploy.input.text_processor import DataProcessor +from fastdeploy.utils import envs + + +MODEL_PATH = "/root/paddlejob/workspace/gzz/ERNIE-4.5-0.3B-Base-PT" + + +class TestDecodeToken(unittest.TestCase): + """Test case for _decode_token method with real tokenizer""" + + def setUp(self): + self.req_id = "test_req_123" + + self.data_processor_obj = DataProcessor(MODEL_PATH) + + self.data_processor = MagicMock() + self.data_processor.tokenizer = self.data_processor_obj.tokenizer + self.data_processor.decode_status = self.data_processor_obj.decode_status + self.data_processor.ids2tokens = self.data_processor_obj.ids2tokens + + self.engine = MagicMock(spec=EngineService) + self.engine.data_processor = self.data_processor + self.engine._decode_token = EngineService._decode_token.__get__(self.engine, EngineService) + + # Common init for decode_status + self.data_processor.decode_status[self.req_id] = [0, 0, [], ""] + + def _tokenize(self, text): + return self.data_processor_obj.tokenizer(text, add_special_tokens=False)['input_ids'] + + def _assert_cleaned_up(self): + self.assertNotIn(self.req_id, self.data_processor.decode_status) + + def test_empty_end(self): + """Empty token_ids with is_end=True should return empty and cleanup""" + with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True): + delta_text, returned_tokens = self.engine._decode_token([], self.req_id, is_end=True) + self.assertEqual(delta_text, "") + self.assertEqual(returned_tokens, []) + self._assert_cleaned_up() + + def test_incremental_decoding_and_cleanup(self): + """Tokens added in multiple steps should decode correctly and cleanup at end""" + with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True): + for char in ["你", "好"]: + tokens = self._tokenize(char) + delta_text, _ = self.engine._decode_token(tokens, self.req_id, is_end=False) + self.assertTrue(len(delta_text) > 0) + + delta_text, _ = self.engine._decode_token([], self.req_id, is_end=True) + self._assert_cleaned_up() + + + + def test_undecoded_tokens_on_end(self): + """Test that tokens which produce no visible text during streaming + are force-decoded when is_end=True""" + with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True), \ + patch.dict(os.environ, {'DEBUG_DECODE': '1'}): + all_delta="" + + delta_text, _ = self.engine._decode_token([109584], self.req_id, is_end=False) + all_delta += delta_text + + + # Now end the stream - force decode should recover any remaining text + delta_end, _ = self.engine._decode_token([109584], self.req_id, is_end=False) + all_delta += delta_end + delta_end, _ = self.engine._decode_token([109584], self.req_id, is_end=False) + all_delta += delta_end + delta_end, token_ids = self.engine._decode_token([], self.req_id, is_end=True) + all_delta += delta_end + + # The full text must be recovered either during streaming or at end + self.assertEqual(token_ids, [109584, 109584, 109584]) + self._assert_cleaned_up() + + + + +if __name__ == "__main__": + unittest.main() From a492d05a126b3669a5ad16f6621fb6e372f62d52 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 31 Mar 2026 17:21:22 +0800 Subject: [PATCH 2/6] case format --- fastdeploy/engine/common_engine.py | 4 ++-- tests/engine/test_decode_token.py | 12 +++--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/fastdeploy/engine/common_engine.py b/fastdeploy/engine/common_engine.py index 989f4681e57..7e74e18717e 100644 --- a/fastdeploy/engine/common_engine.py +++ b/fastdeploy/engine/common_engine.py @@ -1726,11 +1726,11 @@ def _decode_token(self, token_ids, req_id, is_end): token_ids = cum_tokens[prefix_offset:read_offset] else: token_ids = [] - + if is_end and delta_text == "" and len(cum_tokens) > 0: read_offset = self.data_processor.decode_status[req_id][1] token_ids = cum_tokens[read_offset:] - + if is_end: del self.data_processor.decode_status[req_id] return delta_text, token_ids diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py index a9f64a0727e..01f80ab8865 100644 --- a/tests/engine/test_decode_token.py +++ b/tests/engine/test_decode_token.py @@ -25,8 +25,7 @@ from fastdeploy.input.text_processor import DataProcessor from fastdeploy.utils import envs - -MODEL_PATH = "/root/paddlejob/workspace/gzz/ERNIE-4.5-0.3B-Base-PT" +MODEL_PATH = os.getenv("MODEL_PATH","") + "/ERNIE-4.5-0.3B-Paddle" class TestDecodeToken(unittest.TestCase): @@ -74,18 +73,15 @@ def test_incremental_decoding_and_cleanup(self): delta_text, _ = self.engine._decode_token([], self.req_id, is_end=True) self._assert_cleaned_up() - - def test_undecoded_tokens_on_end(self): """Test that tokens which produce no visible text during streaming are force-decoded when is_end=True""" with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True), \ patch.dict(os.environ, {'DEBUG_DECODE': '1'}): - all_delta="" - + all_delta = "" + delta_text, _ = self.engine._decode_token([109584], self.req_id, is_end=False) all_delta += delta_text - # Now end the stream - force decode should recover any remaining text delta_end, _ = self.engine._decode_token([109584], self.req_id, is_end=False) @@ -99,8 +95,6 @@ def test_undecoded_tokens_on_end(self): self.assertEqual(token_ids, [109584, 109584, 109584]) self._assert_cleaned_up() - - if __name__ == "__main__": unittest.main() From b87fd2cd755f743df516a1daa016a10154bef3ce Mon Sep 17 00:00:00 2001 From: root Date: Tue, 31 Mar 2026 18:08:08 +0800 Subject: [PATCH 3/6] case format --- tests/engine/test_decode_token.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py index 01f80ab8865..4c60e815419 100644 --- a/tests/engine/test_decode_token.py +++ b/tests/engine/test_decode_token.py @@ -25,7 +25,7 @@ from fastdeploy.input.text_processor import DataProcessor from fastdeploy.utils import envs -MODEL_PATH = os.getenv("MODEL_PATH","") + "/ERNIE-4.5-0.3B-Paddle" +MODEL_PATH = os.getenv("MODEL_PATH", "") + "/ERNIE-4.5-0.3B-Paddle" class TestDecodeToken(unittest.TestCase): @@ -49,14 +49,14 @@ def setUp(self): self.data_processor.decode_status[self.req_id] = [0, 0, [], ""] def _tokenize(self, text): - return self.data_processor_obj.tokenizer(text, add_special_tokens=False)['input_ids'] + return self.data_processor_obj.tokenizer(text, add_special_tokens=False)["input_ids"] def _assert_cleaned_up(self): self.assertNotIn(self.req_id, self.data_processor.decode_status) def test_empty_end(self): """Empty token_ids with is_end=True should return empty and cleanup""" - with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True): + with patch.object(envs, "FD_ENABLE_RETURN_TEXT", True): delta_text, returned_tokens = self.engine._decode_token([], self.req_id, is_end=True) self.assertEqual(delta_text, "") self.assertEqual(returned_tokens, []) @@ -64,7 +64,7 @@ def test_empty_end(self): def test_incremental_decoding_and_cleanup(self): """Tokens added in multiple steps should decode correctly and cleanup at end""" - with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True): + with patch.object(envs, "FD_ENABLE_RETURN_TEXT", True): for char in ["你", "好"]: tokens = self._tokenize(char) delta_text, _ = self.engine._decode_token(tokens, self.req_id, is_end=False) @@ -76,8 +76,9 @@ def test_incremental_decoding_and_cleanup(self): def test_undecoded_tokens_on_end(self): """Test that tokens which produce no visible text during streaming are force-decoded when is_end=True""" - with patch.object(envs, 'FD_ENABLE_RETURN_TEXT', True), \ - patch.dict(os.environ, {'DEBUG_DECODE': '1'}): + with patch.object( + envs, "FD_ENABLE_RETURN_TEXT", True + ), patch.dict(os.environ, {"DEBUG_DECODE": "1"}): all_delta = "" delta_text, _ = self.engine._decode_token([109584], self.req_id, is_end=False) From 9674030f5aad9435292d80d5b02be3bb109f780e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 31 Mar 2026 19:21:06 +0800 Subject: [PATCH 4/6] case format --- tests/engine/test_decode_token.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py index 4c60e815419..49613382286 100644 --- a/tests/engine/test_decode_token.py +++ b/tests/engine/test_decode_token.py @@ -76,9 +76,7 @@ def test_incremental_decoding_and_cleanup(self): def test_undecoded_tokens_on_end(self): """Test that tokens which produce no visible text during streaming are force-decoded when is_end=True""" - with patch.object( - envs, "FD_ENABLE_RETURN_TEXT", True - ), patch.dict(os.environ, {"DEBUG_DECODE": "1"}): + with patch.object(envs, "FD_ENABLE_RETURN_TEXT", True), patch.dict(os.environ, {"DEBUG_DECODE": "1"}): all_delta = "" delta_text, _ = self.engine._decode_token([109584], self.req_id, is_end=False) From dacec20f462c0d381632d6e0dde0b9db7e4d0c54 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Apr 2026 11:01:38 +0800 Subject: [PATCH 5/6] case format --- tests/engine/test_decode_token.py | 89 ++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 14 deletions(-) diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py index 49613382286..9a4726b66fc 100644 --- a/tests/engine/test_decode_token.py +++ b/tests/engine/test_decode_token.py @@ -1,3 +1,10 @@ +''' +Author: +Date: 2026-03-31 10:40:18 +LastEditors: +LastEditTime: 2026-04-01 11:00:47 +FilePath: /fastdeploy/test_decode_token.py +''' """ # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. # @@ -22,34 +29,85 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) from fastdeploy.engine.common_engine import EngineService -from fastdeploy.input.text_processor import DataProcessor from fastdeploy.utils import envs -MODEL_PATH = os.getenv("MODEL_PATH", "") + "/ERNIE-4.5-0.3B-Paddle" + +def _make_mock_ids2tokens(decode_status, undecoded_tokens=None): + """ + Create a mock ids2tokens that simulates incremental decoding behavior. + + Simulates the non-HF path of DataProcessor.ids2tokens: + - decode_status[task_id] = [prefix_offset, read_offset, cumulative_token_ids, cumulative_text] + - Returns (delta_text, previous_token_ids, previous_texts) + + Args: + decode_status: shared dict for tracking decode state + undecoded_tokens: set of token IDs that produce no visible text (delta_text=""), + simulating tokens that cannot be decoded incrementally. + read_offset will NOT advance for these tokens. + """ + undecoded_tokens = undecoded_tokens or set() + + # Simple token->char mapping for testing + token_map = { + 1000: "你", + 1001: "好", + } + + def mock_ids2tokens(token_ids, task_id): + if task_id not in decode_status: + decode_status[task_id] = [0, 0, [], ""] + + previous_token_ids = list(decode_status[task_id][2]) + previous_texts = decode_status[task_id][3] + + # Append new tokens to cumulative list + decode_status[task_id][2] += token_ids + + # Check if all new tokens are "undecoded" (produce no visible text) + all_undecoded = all(tid in undecoded_tokens for tid in token_ids) if token_ids else True + + if all_undecoded and token_ids: + # These tokens can't be decoded yet - don't advance read_offset + delta_text = "" + else: + # Normal decoding + delta_text = "" + for tid in token_ids: + delta_text += token_map.get(tid, f"<{tid}>") + + if token_ids: + # Only advance offsets when there are actual tokens + cum_len = len(decode_status[task_id][2]) + decode_status[task_id][0] = max(0, cum_len - 1) # prefix_offset + decode_status[task_id][1] = cum_len # read_offset + decode_status[task_id][3] += delta_text + + return delta_text, previous_token_ids, previous_texts + + return mock_ids2tokens class TestDecodeToken(unittest.TestCase): - """Test case for _decode_token method with real tokenizer""" + """Test case for _decode_token method with mocked tokenizer""" def setUp(self): self.req_id = "test_req_123" + self._setup_engine() - self.data_processor_obj = DataProcessor(MODEL_PATH) + def _setup_engine(self, undecoded_tokens=None): + self.decode_status = {} self.data_processor = MagicMock() - self.data_processor.tokenizer = self.data_processor_obj.tokenizer - self.data_processor.decode_status = self.data_processor_obj.decode_status - self.data_processor.ids2tokens = self.data_processor_obj.ids2tokens + self.data_processor.decode_status = self.decode_status + self.data_processor.ids2tokens = _make_mock_ids2tokens(self.decode_status, undecoded_tokens) self.engine = MagicMock(spec=EngineService) self.engine.data_processor = self.data_processor self.engine._decode_token = EngineService._decode_token.__get__(self.engine, EngineService) # Common init for decode_status - self.data_processor.decode_status[self.req_id] = [0, 0, [], ""] - - def _tokenize(self, text): - return self.data_processor_obj.tokenizer(text, add_special_tokens=False)["input_ids"] + self.decode_status[self.req_id] = [0, 0, [], ""] def _assert_cleaned_up(self): self.assertNotIn(self.req_id, self.data_processor.decode_status) @@ -65,9 +123,8 @@ def test_empty_end(self): def test_incremental_decoding_and_cleanup(self): """Tokens added in multiple steps should decode correctly and cleanup at end""" with patch.object(envs, "FD_ENABLE_RETURN_TEXT", True): - for char in ["你", "好"]: - tokens = self._tokenize(char) - delta_text, _ = self.engine._decode_token(tokens, self.req_id, is_end=False) + for token_id in [1000, 1001]: # "你", "好" + delta_text, _ = self.engine._decode_token([token_id], self.req_id, is_end=False) self.assertTrue(len(delta_text) > 0) delta_text, _ = self.engine._decode_token([], self.req_id, is_end=True) @@ -76,6 +133,10 @@ def test_incremental_decoding_and_cleanup(self): def test_undecoded_tokens_on_end(self): """Test that tokens which produce no visible text during streaming are force-decoded when is_end=True""" + # Re-setup with 109584 as an undecoded token (produces no delta_text during streaming) + self._setup_engine(undecoded_tokens={109584}) + self.decode_status[self.req_id] = [0, 0, [], ""] + with patch.object(envs, "FD_ENABLE_RETURN_TEXT", True), patch.dict(os.environ, {"DEBUG_DECODE": "1"}): all_delta = "" From a7a883b428074cb77e95cf3b0d26902288d545a0 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Apr 2026 11:03:48 +0800 Subject: [PATCH 6/6] case format --- tests/engine/test_decode_token.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/engine/test_decode_token.py b/tests/engine/test_decode_token.py index 9a4726b66fc..49ad80b96f9 100644 --- a/tests/engine/test_decode_token.py +++ b/tests/engine/test_decode_token.py @@ -1,10 +1,11 @@ -''' +""" Author: Date: 2026-03-31 10:40:18 -LastEditors: +LastEditors: LastEditTime: 2026-04-01 11:00:47 FilePath: /fastdeploy/test_decode_token.py -''' +""" + """ # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. # @@ -80,7 +81,7 @@ def mock_ids2tokens(token_ids, task_id): # Only advance offsets when there are actual tokens cum_len = len(decode_status[task_id][2]) decode_status[task_id][0] = max(0, cum_len - 1) # prefix_offset - decode_status[task_id][1] = cum_len # read_offset + decode_status[task_id][1] = cum_len # read_offset decode_status[task_id][3] += delta_text return delta_text, previous_token_ids, previous_texts