From dad3c3d7227d9810fe0172a1f117a46d18ed1a29 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 10:16:52 +0800 Subject: [PATCH 1/5] feat: Upgrade pydantic to v2 (#750) 1. Upgrade pydantic to 2.x 2. Remove results/ from .gitignore, those files need to track 3. fix the coding styles in the results Signed-off-by: yangxuan --- .gitignore | 1 - install/requirements_py3.11.txt | 2 +- pyproject.toml | 2 +- .../backend/clients/alisql/config.py | 4 +- .../backend/clients/alloydb/config.py | 22 ++++----- vectordb_bench/backend/clients/api.py | 21 +++++--- .../backend/clients/aws_opensearch/config.py | 25 +++++----- .../backend/clients/chroma/config.py | 2 +- .../backend/clients/cockroachdb/config.py | 8 +-- .../backend/clients/doris/config.py | 9 ++-- .../backend/clients/lindorm/config.py | 28 +++++------ .../backend/clients/mariadb/config.py | 6 +-- .../backend/clients/milvus/config.py | 25 +++++----- .../backend/clients/oss_opensearch/config.py | 49 ++++++++++--------- .../backend/clients/pgdiskann/config.py | 12 ++--- .../backend/clients/pgvecto_rs/config.py | 6 +-- .../backend/clients/pgvector/config.py | 14 +++--- .../backend/clients/pgvectorscale/config.py | 16 +++--- .../backend/clients/polardb/config.py | 2 +- .../backend/clients/qdrant_cloud/config.py | 19 ++++--- vectordb_bench/backend/clients/tidb/config.py | 21 +++++--- vectordb_bench/backend/dataset.py | 26 +++++----- vectordb_bench/base.py | 5 +- .../components/custom/getCustomConfig.py | 6 ++- .../frontend/config/dbCaseConfigs.py | 6 +-- vectordb_bench/frontend/pages/qps_recall.py | 5 +- vectordb_bench/frontend/pages/results.py | 2 +- vectordb_bench/models.py | 6 +-- vectordb_bench/restful/format_res.py | 4 +- vectordb_bench/results/getLeaderboardData.py | 29 ++++++----- .../results/getLeaderboardDataV2.py | 15 +++--- 31 files changed, 213 insertions(+), 185 deletions(-) diff --git a/.gitignore b/.gitignore index 8985eeb4d..cea1306b0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ build/ venv/ .venv/ .idea/ -results/ logs/ # Worktrees diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 4214267a3..130745816 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -20,7 +20,7 @@ psutil polars plotly environs -pydantic=2.0,<3 scikit-learn pymilvus clickhouse_connect diff --git a/pyproject.toml b/pyproject.toml index 2baeb16e3..e72be9697 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "polars", "plotly", "environs", - "pydantic=2.0,<3", "scikit-learn", "pymilvus", # with pandas, numpy "hdrhistogram>=0.10.1", diff --git a/vectordb_bench/backend/clients/alisql/config.py b/vectordb_bench/backend/clients/alisql/config.py index f942c30f1..16c56e90f 100644 --- a/vectordb_bench/backend/clients/alisql/config.py +++ b/vectordb_bench/backend/clients/alisql/config.py @@ -49,8 +49,8 @@ def parse_metric(self) -> str: class AliSQLHNSWConfig(AliSQLIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW def index_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/alloydb/config.py b/vectordb_bench/backend/clients/alloydb/config.py index d6e54e487..11e65084e 100644 --- a/vectordb_bench/backend/clients/alloydb/config.py +++ b/vectordb_bench/backend/clients/alloydb/config.py @@ -43,8 +43,8 @@ class AlloyDBIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class AlloyDBSearchParam(TypedDict): @@ -120,15 +120,15 @@ def _optionally_build_set_options( class AlloyDBScaNNConfig(AlloyDBIndexConfig): index: IndexType = IndexType.SCANN - num_leaves: int | None - quantizer: str | None - enable_pca: str | None - max_num_levels: int | None - num_leaves_to_search: int | None - max_top_neighbors_buffer_size: int | None - pre_reordering_num_neighbors: int | None - num_search_threads: int | None - max_num_prefetch_datasets: int | None + num_leaves: int | None = None + quantizer: str | None = None + enable_pca: str | None = None + max_num_levels: int | None = None + num_leaves_to_search: int | None = None + max_top_neighbors_buffer_size: int | None = None + pre_reordering_num_neighbors: int | None = None + num_search_threads: int | None = None + max_num_prefetch_datasets: int | None = None maintenance_work_mem: str | None = None max_parallel_workers: int | None = None diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 5511f18db..0f8103597 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from enum import StrEnum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, model_validator from vectordb_bench.backend.filter import Filter, FilterOp @@ -90,13 +90,18 @@ def common_long_configs() -> list[str]: def to_dict(self) -> dict: raise NotImplementedError - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in cls.common_short_configs() or field.name in cls.common_long_configs(): - return v - if not v and isinstance(v, str | SecretStr): - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class DBCaseConfig(ABC): diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 5ab63010d..7742d421d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class AWSOS_Engine(Enum): diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py index cd3e01ecc..a1d9903d1 100644 --- a/vectordb_bench/backend/clients/chroma/config.py +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -6,7 +6,7 @@ class ChromaConfig(DBConfig): user: str | None = None - password: SecretStr | None + password: SecretStr | None = None host: SecretStr = "localhost" port: int = 8000 diff --git a/vectordb_bench/backend/clients/cockroachdb/config.py b/vectordb_bench/backend/clients/cockroachdb/config.py index 0d608da8f..88ec0e5ea 100644 --- a/vectordb_bench/backend/clients/cockroachdb/config.py +++ b/vectordb_bench/backend/clients/cockroachdb/config.py @@ -75,16 +75,16 @@ class CockroachDBIndexParam(TypedDict): metric: str index_creation_with_options: Sequence[dict[str, Any]] - min_partition_size: int | None - max_partition_size: int | None - build_beam_size: int | None + min_partition_size: int | None = None + max_partition_size: int | None = None + build_beam_size: int | None = None class CockroachDBSearchParam(TypedDict): """Search parameters for CockroachDB vector queries.""" metric_fun_op: LiteralString - vector_search_beam_size: int | None + vector_search_beam_size: int | None = None class CockroachDBSessionCommands(TypedDict): diff --git a/vectordb_bench/backend/clients/doris/config.py b/vectordb_bench/backend/clients/doris/config.py index a15309922..7c79ba728 100644 --- a/vectordb_bench/backend/clients/doris/config.py +++ b/vectordb_bench/backend/clients/doris/config.py @@ -1,6 +1,6 @@ import logging -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -17,9 +17,10 @@ class DorisConfig(DBConfig): db_name: str = "test" ssl: bool = False - @validator("*") - def not_empty_field(cls, v: any, field: any): - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + return data def to_dict(self) -> dict: pwd_str = self.password.get_secret_value() diff --git a/vectordb_bench/backend/clients/lindorm/config.py b/vectordb_bench/backend/clients/lindorm/config.py index 367f369e3..0e0d4aae6 100644 --- a/vectordb_bench/backend/clients/lindorm/config.py +++ b/vectordb_bench/backend/clients/lindorm/config.py @@ -43,9 +43,9 @@ def parse_metric(self) -> str: class HNSWConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.HNSW - M: int | None - efConstruction: int | None - efSearch: int | None + M: int | None = None + efConstruction: int | None = None + efSearch: int | None = None filter_type: str | None = "efficient_filter" k_expand_scope: int | None = 1000 @@ -72,12 +72,12 @@ def search_param(self, do_filter: bool = False) -> dict: # first layer searching for cluster centroids is hnsw class IVFPQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFPQ - nlist: int | None - nprobe: int | None + nlist: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 @@ -116,13 +116,13 @@ def search_param(self, do_filter: bool = False) -> dict: class IVFBQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFBQ - nlist: int | None - exbits: int | None - nprobe: int | None + nlist: int | None = None + exbits: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py index d183adc76..21ea9ac2e 100644 --- a/vectordb_bench/backend/clients/mariadb/config.py +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -46,11 +46,11 @@ def parse_metric(self) -> str: class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW storage_engine: str = "InnoDB" - max_cache_size: int | None + max_cache_size: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 9ffbdcece..98118c6df 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType @@ -19,17 +19,18 @@ def to_dict(self) -> dict: "replica_number": self.replica_number, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class MilvusIndexConfig(BaseModel): diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index 83fed3d58..a5d69459a 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, root_validator, validator +from pydantic import BaseModel, SecretStr, field_validator, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class OSSOS_Engine(Enum): @@ -111,7 +112,8 @@ class OSSOpenSearchIndexConfig(BaseModel, DBCaseConfig): compression_level: str = CompressionLevel.LEVEL_32X oversample_factor: float = 1.0 - @validator("quantization_type", pre=True, always=True) + @field_validator("quantization_type", mode="before") + @classmethod def validate_quantization_type(cls, value: any): """Convert string values to enum""" if not value: @@ -128,19 +130,22 @@ def validate_quantization_type(cls, value: any): return mapping.get(value, OSSOpenSearchQuantization.NONE) - @root_validator - def validate_engine_name(cls, values: dict): - """Map engine_name string from UI to engine enum""" - if values.get("engine_name"): - engine_name = values["engine_name"].lower() + @model_validator(mode="before") + @classmethod + def validate_engine_name(cls, data: any) -> any: + if not isinstance(data, dict): + return data + # Map engine_name to engine enum + if data.get("engine_name"): + engine_name = data["engine_name"].lower() if engine_name == "faiss": - values["engine"] = OSSOS_Engine.faiss + data["engine"] = OSSOS_Engine.faiss elif engine_name == "lucene": - values["engine"] = OSSOS_Engine.lucene + data["engine"] = OSSOS_Engine.lucene else: log.warning(f"Unknown engine_name: {engine_name}, defaulting to faiss") - values["engine"] = OSSOS_Engine.faiss - return values + data["engine"] = OSSOS_Engine.faiss + return data def __eq__(self, obj: any): return ( diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py index 7f83a05c8..8715b1e42 100644 --- a/vectordb_bench/backend/clients/pgdiskann/config.py +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -43,8 +43,8 @@ class PgDiskANNIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None create_index_before_load: bool = False create_index_after_load: bool = True - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -120,10 +120,10 @@ def _optionally_build_set_options( class PgDiskANNImplConfig(PgDiskANNIndexConfig): index: IndexType = IndexType.DISKANN - max_neighbors: int | None - l_value_ib: int | None - pq_param_num_chunks: int | None - l_value_is: float | None + max_neighbors: int | None = None + l_value_ib: int | None = None + pq_param_num_chunks: int | None = None + l_value_is: float | None = None reranking: bool | None = None reranking_metric: str | None = None quantized_fetch_limit: int | None = None diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index fbb7c5d81..73c2573a0 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -78,7 +78,7 @@ def session_param(self) -> dict[str, str | int]: ... class PgVectoRSHNSWConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.HNSW m: int | None = None - ef_search: int | None + ef_search: int | None = None ef_construction: int | None = None def index_param(self) -> dict[str, str]: @@ -106,8 +106,8 @@ def session_param(self) -> dict[str, str | int]: class PgVectoRSIVFFlatConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.IVFFlat - probes: int | None - lists: int | None + probes: int | None = None + lists: int | None = None def index_param(self) -> dict[str, str]: if self.quantization_type is None: diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 98e82f1c2..7da238a4b 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -47,8 +47,8 @@ class PgVectorIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class PgVectorSearchParam(TypedDict): @@ -175,13 +175,13 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): a good place to start is sqrt(lists) """ - lists: int | None - probes: int | None + lists: int | None = None + probes: int | None = None index: IndexType = IndexType.ES_IVFFlat maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None @@ -226,12 +226,12 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): m: int | None # DETAIL: Valid values are between "2" and "100". ef_construction: int | None # ef_construction must be greater than or equal to 2 * m - ef_search: int | None + ef_search: int | None = None index: IndexType = IndexType.ES_HNSW maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None diff --git a/vectordb_bench/backend/clients/pgvectorscale/config.py b/vectordb_bench/backend/clients/pgvectorscale/config.py index e22c45c8d..07750cffb 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/config.py +++ b/vectordb_bench/backend/clients/pgvectorscale/config.py @@ -70,14 +70,14 @@ def session_param(self) -> dict: ... class PgVectorScaleStreamingDiskANNConfig(PgVectorScaleIndexConfig): index: IndexType = IndexType.STREAMING_DISKANN - storage_layout: str | None - num_neighbors: int | None - search_list_size: int | None - max_alpha: float | None - num_dimensions: int | None - num_bits_per_dimension: int | None - query_search_list_size: int | None - query_rescore: int | None + storage_layout: str | None = None + num_neighbors: int | None = None + search_list_size: int | None = None + max_alpha: float | None = None + num_dimensions: int | None = None + num_bits_per_dimension: int | None = None + query_search_list_size: int | None = None + query_rescore: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/polardb/config.py b/vectordb_bench/backend/clients/polardb/config.py index c75448c49..5121c1e19 100644 --- a/vectordb_bench/backend/clients/polardb/config.py +++ b/vectordb_bench/backend/clients/polardb/config.py @@ -11,7 +11,7 @@ class PolarDBConfigDict(TypedDict): host: str port: int database: str - unix_socket: str | None + unix_socket: str | None = None class PolarDBConfig(DBConfig): diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index b2eeb2ce6..06543aaab 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,6 +1,6 @@ from typing import TypeVar -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -25,11 +25,18 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["api_key"]: - return v - return super().not_empty_field(v, field) + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"api_key"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class QdrantIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 71fdbad66..93098ede1 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,6 +1,6 @@ from typing import TypedDict -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -35,13 +35,18 @@ def to_dict(self) -> TiDBConfigDict: "ssl_verify_identity": self.ssl, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["password", "db_label"]: - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class TiDBIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d1de9e328..94216532f 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -7,12 +7,12 @@ import logging import pathlib from enum import Enum -from typing import Any, NamedTuple +from typing import Any, ClassVar, NamedTuple import pandas as pd import polars as pl from pyarrow.parquet import ParquetFile -from pydantic import PrivateAttr, validator +from pydantic import field_validator from vectordb_bench import config from vectordb_bench.base import BaseModel @@ -38,7 +38,7 @@ class BaseDataset(BaseModel): metric_type: MetricType use_shuffled: bool with_gt: bool = False - _size_label: dict[int, SizeLabel] = PrivateAttr() + _size_label: ClassVar[dict[int, SizeLabel]] is_custom: bool = False with_remote_resource: bool = True # for label filter cases @@ -57,7 +57,8 @@ class BaseDataset(BaseModel): gt_id_field: str = "id" gt_neighbors_field: str = "neighbors_id" - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): if v not in cls._size_label: msg = f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}" @@ -102,7 +103,8 @@ class CustomDataset(BaseDataset): scalar_labels_file: str = "scalar_labels.parquet" label_percentages: list[float] = [] - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): return v @@ -136,7 +138,7 @@ class LAION(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -146,7 +148,7 @@ class GIST(BaseDataset): dim: int = 960 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), } @@ -158,7 +160,7 @@ class Cohere(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), @@ -196,7 +198,7 @@ class Bioasq(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } @@ -232,7 +234,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} + _size_label: ClassVar[dict] = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} class SIFT(BaseDataset): @@ -240,7 +242,7 @@ class SIFT(BaseDataset): dim: int = 128 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 500_000: SizeLabel( 500_000, "SMALL", @@ -257,7 +259,7 @@ class OpenAI(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), diff --git a/vectordb_bench/base.py b/vectordb_bench/base.py index 502d5fa49..401d2086d 100644 --- a/vectordb_bench/base.py +++ b/vectordb_bench/base.py @@ -1,5 +1,6 @@ from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict -class BaseModel(PydanticBaseModel, arbitrary_types_allowed=True): - pass +class BaseModel(PydanticBaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/vectordb_bench/frontend/components/custom/getCustomConfig.py b/vectordb_bench/frontend/components/custom/getCustomConfig.py index a1ddfb737..03831a83c 100644 --- a/vectordb_bench/frontend/components/custom/getCustomConfig.py +++ b/vectordb_bench/frontend/components/custom/getCustomConfig.py @@ -62,14 +62,16 @@ def get_custom_streaming_configs(): def save_custom_configs(custom_configs: list[CustomDatasetConfig]): with open(config.CUSTOM_CONFIG_DIR, "w") as f: - json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4) + json.dump([custom_config.model_dump() for custom_config in custom_configs], f, indent=4) def save_all_custom_configs( performance_configs: list[CustomCaseConfig], streaming_configs: list[CustomStreamingCaseConfig] ): """Save both performance and streaming configs to the same JSON file""" - all_configs = [config.dict() for config in performance_configs] + [config.dict() for config in streaming_configs] + all_configs = [config.model_dump() for config in performance_configs] + [ + config.model_dump() for config in streaming_configs + ] with open(config.CUSTOM_CONFIG_DIR, "w") as f: json.dump(all_configs, f, indent=4) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 387f7fb4a..d15c4e7ee 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -119,7 +119,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": False, }, ) @@ -140,7 +140,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": True, "label_percentage": label_percentage, }, @@ -174,7 +174,7 @@ def get_custom_streaming_case_items() -> list[UICaseItem]: case_id=CaseType.StreamingCustomDataset, custom_case={ "description": custom_config.description, - "dataset_config": custom_config.dataset_config.dict(), + "dataset_config": custom_config.dataset_config.model_dump(), }, ) ], diff --git a/vectordb_bench/frontend/pages/qps_recall.py b/vectordb_bench/frontend/pages/qps_recall.py index 27f9c4691..fb8f680c5 100644 --- a/vectordb_bench/frontend/pages/qps_recall.py +++ b/vectordb_bench/frontend/pages/qps_recall.py @@ -43,7 +43,10 @@ def case_results_filter(case_result: CaseResult) -> bool: case = case_result.task_config.case_config.case return case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter - default_selected_task_labels = ["standard_2025"] + default_selected_task_labels = ["standard_20260403", "standard_20250519"] + # Filter defaults to only include labels that exist in results + available_labels = {r.task_label for r in allResults} + default_selected_task_labels = [l for l in default_selected_task_labels if l in available_labels] shownData, failedTasks, showCaseNames = getshownData( resultSelectorContainer, allResults, diff --git a/vectordb_bench/frontend/pages/results.py b/vectordb_bench/frontend/pages/results.py index a146f2fdc..216029bb1 100644 --- a/vectordb_bench/frontend/pages/results.py +++ b/vectordb_bench/frontend/pages/results.py @@ -32,7 +32,7 @@ def main(): st.caption( "Choose your desired test results to display from the sidebar. " "For your reference, we've included two standard benchmarks tested by our team. " - "Note that `standard_2025` was tested in 2025; the others in 2023. " + "Note that `standard_20260403` is the latest benchmark; the others were tested in 2023-2025. " "Unless explicitly labeled as distributed multi-node, test with single-node mode by default." ) st.caption("We welcome community contributions for better results, parameter configurations, and optimizations.") diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index cdc64b9d7..a7e7c09f1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -205,7 +205,7 @@ def k(self, value): ''' def __hash__(self) -> int: - return hash(self.json()) + return hash(self.model_dump_json()) @property def case(self) -> Case: @@ -314,7 +314,7 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"write results to disk {result_file}") with pathlib.Path(result_file).open("w") as f: - b = partial.json(exclude={"db_config": {"password", "api_key"}}) + b = partial.model_dump_json(exclude={"db_config": {"password", "api_key"}}) f.write(b) def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: @@ -381,7 +381,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: else: # Default to 0 for older result files that don't have P95 data case_result["metrics"]["serial_latency_p95"] = 0.0 - return TestResult.validate(test_result) + return TestResult.model_validate(test_result) def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py index 2e289ec3b..326986319 100644 --- a/vectordb_bench/restful/format_res.py +++ b/vectordb_bench/restful/format_res.py @@ -63,7 +63,7 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict db_label=task_config.db_config.db_label, version=task_config.db_config.version, note=task_config.db_config.note, - params=task_config.db_case_config.dict(), + params=task_config.db_case_config.model_dump(), case_name=case.name, dataset=dataset.full_name, dim=dataset.dim, @@ -71,6 +71,6 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict filter_rate=filter_.filter_rate, k=task_config.case_config.k, **metrics, - ).dict() + ).model_dump() ) return results diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index aef024bdc..1650a6f87 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -1,14 +1,16 @@ -from vectordb_bench import config -import ujson import pathlib +from datetime import datetime + +import ujson + +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import ResultLabel, TestResult -from datetime import datetime -taskLabelToCode = { +task_label_to_code = { ResultLabel.FAILED: -1, ResultLabel.OUTOFRANGE: -2, ResultLabel.NORMAL: 1, @@ -18,15 +20,14 @@ def format_time(ts: float) -> str: default_standard_test_time = datetime(2023, 8, 1) t = datetime.fromtimestamp(ts) - if t < default_standard_test_time: - t = default_standard_test_time + t = max(t, default_standard_test_time) return t.strftime("%Y-%m") def main(): - allResults: list[TestResult] = benchMarkRunner.get_results() + all_results: list[TestResult] = benchMarkRunner.get_results() - if allResults is not None: + if all_results is not None: data = [ { "db": d.task_config.db.value, @@ -36,18 +37,16 @@ def main(): "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, "recall": d.metrics.recall, - "label": taskLabelToCode[d.label], + "label": task_label_to_code[d.label], "note": d.task_config.db_config.note, "version": d.task_config.db_config.version, "test_time": format_time(test_result.timestamp), } - for test_result in allResults + for test_result in all_results if "standard" in test_result.task_label for d in test_result.results - if d.task_config.case_config.case_id != CaseType.CapacityDim128 - and d.task_config.case_config.case_id != CaseType.CapacityDim960 - if d.task_config.db != DB.ZillizCloud - or test_result.timestamp >= datetime(2024, 1, 1).timestamp() + if d.task_config.case_config.case_id not in {CaseType.CapacityDim128, CaseType.CapacityDim960} + if d.task_config.db != DB.ZillizCloud or test_result.timestamp >= datetime(2024, 1, 1).timestamp() ] # compute qp$ @@ -58,7 +57,7 @@ def main(): price = DB_DBLABEL_TO_PRICE.get(db, {}).get(db_label, 0) d["qp$"] = (qps / price * 3600) if price > 0 else 0.0 - with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json"), "w") as f: + with pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json").open("w") as f: ujson.dump(data, f) diff --git a/vectordb_bench/results/getLeaderboardDataV2.py b/vectordb_bench/results/getLeaderboardDataV2.py index 62440886f..188d9876f 100644 --- a/vectordb_bench/results/getLeaderboardDataV2.py +++ b/vectordb_bench/results/getLeaderboardDataV2.py @@ -1,17 +1,14 @@ import json import logging +import pathlib - +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase -from vectordb_bench.backend.clients import DB +from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseResult -from vectordb_bench import config -import numpy as np logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s") -from vectordb_bench.interface import BenchMarkRunner - def get_standard_2025_results() -> list[CaseResult]: all_results = BenchMarkRunner.get_results() @@ -23,7 +20,7 @@ def get_standard_2025_results() -> list[CaseResult]: def save_to_json(data: list[dict], file_name: str): - with open(file_name, "w") as f: + with pathlib.Path(file_name).open("w") as f: json.dump(data, f, indent=4) @@ -56,11 +53,11 @@ def main(): } ) else: - case: StreamingPerformanceCase = case + streaming_case: StreamingPerformanceCase = case # use 90p search stage results to represent streaming performance qps_90p = metrics.st_max_qps_list_list[metrics.st_search_stage_list.index(90)] latency_90p = metrics.st_serial_latency_p99_list[metrics.st_search_stage_list.index(90)] - insert_rate = case.insert_rate + insert_rate = streaming_case.insert_rate streaming_data.append( { "dataset": dataset, From cf09d634edafcd0379a707b72873b7a8879e30dc Mon Sep 17 00:00:00 2001 From: ChenLiqing Date: Thu, 9 Apr 2026 11:01:38 +0800 Subject: [PATCH 2/5] fix: fill missing build durations in Milvus and ZillizCloud results (#751) Populate insert_duration, optimize_duration, load_duration for all entries in result_20260403 files. Previously only the first entry per index had values while the rest were 0.0. Milvus (re-measured on 2.6-opt-v2): - 1M SQ4U: insert=129.8s, optimize=152.2s, load=282.0s - 1M SQ8: insert=119.5s, optimize=235.9s, load=355.4s - 10M SQ4U/SQ8: copied from existing first-entry values ZillizCloud (from prior build runs): - 1M: insert=246.7s, optimize=101.2s, load=347.9s - 10M: insert=2450.8s, optimize=136.9s, load=2587.8s Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.6 (1M context) --- .../result_20260403_standard_milvus.json | 180 +++++++++--------- .../result_20260403_standard_zillizcloud.json | 108 +++++------ 2 files changed, 144 insertions(+), 144 deletions(-) diff --git a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json index e10404530..56219eb1d 100644 --- a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json +++ b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json @@ -134,9 +134,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3628.8527, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -261,9 +261,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3250.1112, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -388,9 +388,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2762.4144, "serial_latency_p99": 0.0031, "serial_latency_p95": 0.0029, @@ -515,9 +515,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2384.6245, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -642,9 +642,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2134.1717, "serial_latency_p99": 0.0038, "serial_latency_p95": 0.0036, @@ -769,9 +769,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1641.3478, "serial_latency_p99": 0.0041, "serial_latency_p95": 0.0039, @@ -896,9 +896,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1488.5841, "serial_latency_p99": 0.0047, "serial_latency_p95": 0.0043, @@ -1152,9 +1152,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2514.4481, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -1279,9 +1279,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2177.2345, "serial_latency_p99": 0.0034, "serial_latency_p95": 0.0031, @@ -1406,9 +1406,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1833.2575, "serial_latency_p99": 0.0039, "serial_latency_p95": 0.0035, @@ -1533,9 +1533,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1552.4803, "serial_latency_p99": 0.004, "serial_latency_p95": 0.0037, @@ -1660,9 +1660,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1355.3121, "serial_latency_p99": 0.0044, "serial_latency_p95": 0.0042, @@ -1787,9 +1787,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1079.2123, "serial_latency_p99": 0.0053, "serial_latency_p95": 0.0049, @@ -1914,9 +1914,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 876.5772, "serial_latency_p99": 0.0063, "serial_latency_p95": 0.0059, @@ -2041,9 +2041,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10663.1231, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0018, @@ -2168,9 +2168,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10333.9072, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -2295,9 +2295,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 9575.6863, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0021, @@ -2422,9 +2422,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 8596.7694, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0022, @@ -2549,9 +2549,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7704.3625, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -2676,9 +2676,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7023.6735, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0028, @@ -2803,9 +2803,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 6031.3725, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.003, @@ -2930,9 +2930,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 5258.1868, "serial_latency_p99": 0.0036, "serial_latency_p95": 0.0033, @@ -3057,9 +3057,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5973.0024, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -3184,9 +3184,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5416.5758, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -3311,9 +3311,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4771.4324, "serial_latency_p99": 0.0028, "serial_latency_p95": 0.0025, @@ -3438,9 +3438,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4006.3994, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -3565,9 +3565,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3441.7597, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0032, @@ -3692,9 +3692,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3040.6216, "serial_latency_p99": 0.0037, "serial_latency_p95": 0.0035, @@ -3819,9 +3819,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2446.7373, "serial_latency_p99": 0.0043, "serial_latency_p95": 0.004, @@ -3946,9 +3946,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2084.6245, "serial_latency_p99": 0.005, "serial_latency_p95": 0.0046, diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index e02463ebd..fe81ffe05 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032, From 268e7ab3d8e282af3f422c772f1db81f12c3d16b Mon Sep 17 00:00:00 2001 From: Alexander Guzhva Date: Thu, 9 Apr 2026 07:03:18 +0000 Subject: [PATCH 3/5] Introduce Intel SVS (#749) Signed-off-by: Alexandr Guzhva --- vectordb_bench/backend/clients/api.py | 3 + vectordb_bench/backend/clients/milvus/cli.py | 163 ++++++++++++++++++ .../backend/clients/milvus/config.py | 62 +++++++ 3 files changed, 228 insertions(+) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 0f8103597..ecbddef39 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -43,6 +43,9 @@ class IndexType(StrEnum): GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" SCANN_MILVUS = "SCANN_MILVUS" + SVS_VAMANA = "SVS_VAMANA" + SVS_VAMANA_LVQ = "SVS_VAMANA_LVQ" + SVS_VAMANA_LEANVEC = "SVS_VAMANA_LEANVEC" Hologres_HGraph = "HGraph" Hologres_Graph = "Graph" NONE = "NONE" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 2f2a286be..ae7269801 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -485,6 +485,169 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): ) +class MilvusSVSVamanaTypedDict(CommonTypedDict, MilvusTypedDict): + svs_graph_max_degree: Annotated[ + int, + click.option( + "--svs-graph-max-degree", + type=int, + help="Maximum degree of the Vamana graph (4-256).", + required=True, + ), + ] + svs_construction_window_size: Annotated[ + int, + click.option( + "--svs-construction-window-size", + type=int, + help="Window size for graph construction.", + required=False, + default=40, + show_default=True, + ), + ] + svs_alpha: Annotated[ + float | None, + click.option( + "--svs-alpha", + type=float, + help="Pruning parameter (default: 1.2 for L2, 0.95 for IP/COSINE).", + required=False, + default=None, + ), + ] + svs_storage_kind: Annotated[ + str, + click.option( + "--svs-storage-kind", + type=click.Choice( + ["fp32", "fp16", "sqi8", "lvq4x0", "lvq4x4", "lvq4x8", "leanvec4x4", "leanvec4x8", "leanvec8x8"], + case_sensitive=False, + ), + help="Data storage format.", + required=False, + default="fp32", + show_default=True, + ), + ] + svs_search_window_size: Annotated[ + int | None, + click.option( + "--svs-search-window-size", + type=int, + help="Window size for search (1-10000).", + required=False, + default=None, + ), + ] + svs_search_buffer_capacity: Annotated[ + int | None, + click.option( + "--svs-search-buffer-capacity", + type=int, + help="Buffer capacity for search priority queue (1-10000).", + required=False, + default=None, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaLVQConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLVQConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +class MilvusSVSVamanaLeanVecTypedDict(MilvusSVSVamanaTypedDict): + svs_leanvec_dim: Annotated[ + int, + click.option( + "--svs-leanvec-dim", + type=int, + help="Dimensionality for LeanVec compression (0 = d/2).", + required=False, + default=0, + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaLeanVecTypedDict) +def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict]): + from .config import MilvusConfig, SVSVamanaLeanVecConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLeanVecConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + svs_leanvec_dim=parameters["svs_leanvec_dim"], + ), + **parameters, + ) + + class MilvusGPUIVFPQTypedDict( CommonTypedDict, MilvusTypedDict, diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 98118c6df..620a6b484 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -442,6 +442,65 @@ def search_param(self) -> dict: } +class SVSVamanaConfig(MilvusIndexConfig, DBCaseConfig): + svs_graph_max_degree: int + svs_construction_window_size: int = 40 + svs_alpha: float | None = None + svs_storage_kind: str = "fp32" + svs_search_window_size: int | None = None + svs_search_buffer_capacity: int | None = None + index: IndexType = IndexType.SVS_VAMANA + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": { + "svs_search_window_size": self.svs_search_window_size, + "svs_search_buffer_capacity": self.svs_search_buffer_capacity, + }, + } + + +class SVSVamanaLVQConfig(SVSVamanaConfig): + svs_storage_kind: str = "lvq4x4" + index: IndexType = IndexType.SVS_VAMANA_LVQ + + +class SVSVamanaLeanVecConfig(SVSVamanaConfig): + svs_storage_kind: str = "leanvec4x4" + svs_leanvec_dim: int = 0 + index: IndexType = IndexType.SVS_VAMANA_LEANVEC + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + "svs_leanvec_dim": self.svs_leanvec_dim, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, @@ -459,4 +518,7 @@ def search_param(self) -> dict: IndexType.GPU_CAGRA: GPUCAGRAConfig, IndexType.GPU_BRUTE_FORCE: GPUBruteForceConfig, IndexType.SCANN_MILVUS: SCANNConfig, + IndexType.SVS_VAMANA: SVSVamanaConfig, + IndexType.SVS_VAMANA_LVQ: SVSVamanaLVQConfig, + IndexType.SVS_VAMANA_LEANVEC: SVSVamanaLeanVecConfig, } From 619ce1bb82bec5aa05956f45ec0c70ba59c8366c Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 15:37:04 +0800 Subject: [PATCH 4/5] fix: Skip compaction when encouters permission error (#753) Signed-off-by: yangxuan --- vectordb_bench/backend/clients/milvus/milvus.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 9e9dfb7f9..d36a15c24 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -165,23 +165,27 @@ def _optimize(self): log.info(f"{self.name} optimizing before search") try: self.client.flush(self.collection_name) - self._wait_for_segments_sorted() - self._wait_for_index() + if self.case_config.is_gpu_index: log.debug("skip force merge compaction for gpu index type.") else: try: + # wait for sort, index, compact + self._wait_for_segments_sorted() + self._wait_for_index() compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) if compaction_id > 0: self._wait_for_compaction(compaction_id) log.info(f"{self.name} force merge compaction completed.") - self._wait_for_index() except Exception as e: - log.warning(f"{self.name} compact error: {e}") + log.warning(f"{self.name} compact or list segments error: {e}") if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": - log.warning("Skip compact due to permission denied.") + log.warning("Skip compact due to list segments or compact permission denied.") else: raise e from None + + # wait for index no matter what + self._wait_for_index() self.client.refresh_load(self.collection_name) except Exception as e: log.warning(f"{self.name} optimize error: {e}") From d503114918a3a4b710ad10c6a01d037c59bb88f0 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 13 Apr 2026 06:32:56 +0000 Subject: [PATCH 5/5] fix: refresh ZillizCloud build durations for 1M and 10M baselines Update result_20260403_standard_zillizcloud.json to use the latest validated build timings from recent reruns for case_id=5 (1M) and case_id=4 (10M), including insert_duration, optimize_duration, and load_duration. Co-Authored-By: Claude Sonnet 4.6 --- .../result_20260403_standard_zillizcloud.json | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index fe81ffe05..1f18a729a 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032,