From 725feefe9080e7b8a10c18235ca321eafe2a12ca Mon Sep 17 00:00:00 2001 From: liyonghua0910 Date: Tue, 19 May 2026 08:36:45 +0000 Subject: [PATCH] [Metric] Add free_cpu_block_num gauge metric Add free_cpu_block_num to track available CPU blocks in cache, complementing the existing free_gpu_block_num metric. This enables monitoring CPU block usage for swap/prefix caching scenarios. Co-Authored-By: Claude Opus 4.6 --- docs/online_serving/metrics.md | 1 + docs/zh/online_serving/metrics.md | 1 + fastdeploy/cache_manager/prefix_cache_manager.py | 6 ++++++ fastdeploy/metrics/metrics.py | 7 +++++++ 4 files changed, 15 insertions(+) diff --git a/docs/online_serving/metrics.md b/docs/online_serving/metrics.md index 8e2dd286888..8fefa3f339f 100644 --- a/docs/online_serving/metrics.md +++ b/docs/online_serving/metrics.md @@ -38,6 +38,7 @@ After FastDeploy is launched, it supports continuous monitoring of the FastDeplo | KV Cache | `fastdeploy:free_gpu_block_num` | Gauge | Number of free GPU blocks in cache | count | | KV Cache | `fastdeploy:max_gpu_block_num` | Gauge | Total number of GPU blocks initialized at startup | count | | KV Cache | `fastdeploy:max_cpu_block_num` | Gauge | Total number of CPU blocks initialized at startup | count | +| KV Cache | `fastdeploy:free_cpu_block_num` | Gauge | Number of free CPU blocks in cache | count | | KV Cache | `fastdeploy:available_gpu_resource` | Gauge | Ratio of available GPU blocks to total GPU blocks | % | | KV Cache | `fastdeploy:gpu_cache_usage_perc` | Gauge | GPU KV cache utilization | % | | KV Cache | `fastdeploy:send_cache_failed_num` | Counter | Total number of cache send failures | count | diff --git a/docs/zh/online_serving/metrics.md b/docs/zh/online_serving/metrics.md index 630f68e2ff8..d526077fcb5 100644 --- a/docs/zh/online_serving/metrics.md +++ b/docs/zh/online_serving/metrics.md @@ -38,6 +38,7 @@ | KV缓存 | `fastdeploy:free_gpu_block_num` | Gauge | 缓存中的可用块数 | 个 | | KV缓存 | `fastdeploy:max_gpu_block_num` | Gauge | 服务启动时确定的 GPU 总块数 | 个 | | KV缓存 | `fastdeploy:max_cpu_block_num` | Gauge | 服务启动时确定的 CPU 总块数 | 个 | +| KV缓存 | `fastdeploy:free_cpu_block_num` | Gauge | 缓存中的可用 CPU 块数 | 个 | | KV缓存 | `fastdeploy:available_gpu_resource` | Gauge | 可用块占比，即可用 GPU 块数量 / 最大GPU块数量| 百分比 | | KV缓存 | `fastdeploy:gpu_cache_usage_perc` | Gauge | GPU 上的 KV 缓存使用率 | 百分比 | | KV缓存 | `fastdeploy:send_cache_failed_num` | Counter | 发送缓存失败的总次数 | 个 | diff --git a/fastdeploy/cache_manager/prefix_cache_manager.py b/fastdeploy/cache_manager/prefix_cache_manager.py index f30274ceaed..2f18f3846cd 100644 --- a/fastdeploy/cache_manager/prefix_cache_manager.py +++ b/fastdeploy/cache_manager/prefix_cache_manager.py @@ -132,6 +132,7 @@ def __init__( main_process_metrics.max_cpu_block_num.set(self.num_cpu_blocks) main_process_metrics.available_gpu_block_num.set(self.num_gpu_blocks) main_process_metrics.free_gpu_block_num.set(self.num_gpu_blocks) + main_process_metrics.free_cpu_block_num.set(self.num_cpu_blocks) main_process_metrics.available_gpu_resource.set(1.0) def _get_kv_cache_shape(self, max_block_num): @@ -461,6 +462,7 @@ def update_cache_config(self, cache_config): main_process_metrics.max_cpu_block_num.set(self.num_cpu_blocks) main_process_metrics.available_gpu_block_num.set(self.num_gpu_blocks) main_process_metrics.free_gpu_block_num.set(self.num_gpu_blocks) + main_process_metrics.free_cpu_block_num.set(self.num_cpu_blocks) main_process_metrics.available_gpu_resource.set(1.0) def can_allocate_gpu_blocks(self, num_blocks: int): @@ -520,6 +522,7 @@ def allocate_cpu_blocks(self, num_blocks): logger.info( f"allocate_cpu_blocks: {allocated_block_ids}, len(self.cpu_free_block_list) {len(self.cpu_free_block_list)}" ) + main_process_metrics.free_cpu_block_num.set(len(self.cpu_free_block_list)) return allocated_block_ids def recycle_cpu_blocks(self, cpu_block_ids): @@ -534,6 +537,7 @@ def recycle_cpu_blocks(self, cpu_block_ids): heapq.heappush(self.cpu_free_block_list, cpu_block_id) else: heapq.heappush(self.cpu_free_block_list, cpu_block_ids) + main_process_metrics.free_cpu_block_num.set(len(self.cpu_free_block_list)) def issue_swap_task( self, @@ -1439,6 +1443,7 @@ def free_cpu_block_ids(self, need_block_num): logger.info( "free_cpu_block_ids: after free, " + f"len(self.cpu_free_block_list) {len(self.cpu_free_block_list)}" ) + main_process_metrics.free_cpu_block_num.set(len(self.cpu_free_block_list)) return total_cpu_free_count def get_block_hash_extra_keys(self, request, start_idx, end_idx, mm_idx): @@ -2127,6 +2132,7 @@ def reset(self): # reset metrics self.metrics.reset_metrics() main_process_metrics.free_gpu_block_num.set(len(self.gpu_free_block_list)) + main_process_metrics.free_cpu_block_num.set(len(self.cpu_free_block_list)) main_process_metrics.available_gpu_block_num.set(len(self.gpu_free_block_list)) main_process_metrics.available_gpu_resource.set(self.available_gpu_resource) diff --git a/fastdeploy/metrics/metrics.py b/fastdeploy/metrics/metrics.py index 2384e57c7b3..d5105f20f86 100644 --- a/fastdeploy/metrics/metrics.py +++ b/fastdeploy/metrics/metrics.py @@ -159,6 +159,7 @@ class MetricsManager: free_gpu_block_num: "Gauge" max_gpu_block_num: "Gauge" max_cpu_block_num: "Gauge" + free_cpu_block_num: "Gauge" available_gpu_resource: "Gauge" requests_number: "Counter" send_cache_failed_num: "Counter" @@ -245,6 +246,12 @@ class MetricsManager: "description": "Number of total CPU blocks determined when service started", "kwargs": {}, }, + "free_cpu_block_num": { + "type": Gauge, + "name": "fastdeploy:free_cpu_block_num", + "description": "Number of free CPU blocks in cache", + "kwargs": {}, + }, "available_gpu_resource": { "type": Gauge, "name": "fastdeploy:available_gpu_resource",