From 452f13f2624254a380f71b73ddc67e104f4d8868 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 14 Aug 2026 11:26:48 +0800 Subject: [PATCH] [doc](lakehouse) Add external metadata cache memory management for 4.1.4 --- .../catalog_meta_cache_statistics.md | 93 +++--- .../lakehouse/catalogs/hive-catalog.mdx | 9 +- .../lakehouse/catalogs/hudi-catalog.md | 4 +- .../lakehouse/catalogs/iceberg-catalog.mdx | 14 +- .../lakehouse/catalogs/maxcompute-catalog.md | 4 +- .../lakehouse/catalogs/paimon-catalog.mdx | 14 +- .../external-meta-cache-memory-management.md | 295 ++++++++++++++++++ .../version-4.x/lakehouse/meta-cache.md | 2 + .../catalog_meta_cache_statistics.md | 96 +++--- .../lakehouse/catalogs/hive-catalog.mdx | 9 +- .../lakehouse/catalogs/hudi-catalog.md | 4 +- .../lakehouse/catalogs/iceberg-catalog.mdx | 13 +- .../lakehouse/catalogs/maxcompute-catalog.md | 4 +- .../lakehouse/catalogs/paimon-catalog.mdx | 14 +- .../external-meta-cache-memory-management.md | 289 +++++++++++++++++ .../version-4.x/lakehouse/meta-cache.md | 2 + versioned_sidebars/version-4.x-sidebars.json | 1 + 17 files changed, 750 insertions(+), 117 deletions(-) create mode 100644 i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/external-meta-cache-memory-management.md create mode 100644 versioned_docs/version-4.x/lakehouse/external-meta-cache-memory-management.md diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md index 4aa3cff765b86..14311e1aad210 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md @@ -2,63 +2,80 @@ { "title": "catalog_meta_cache_statistics", "language": "zh-CN", - "description": "查看当前连接的 FE 中，External Catalog 的元数据缓存信息。" + "description": "查看 FE 节点上的 External Catalog 元数据缓存配置、运行状态和内存治理统计信息。" } --- ## 概述 -查看当前连接的 FE 中，External Catalog 的元数据缓存信息。 +`catalog_meta_cache_statistics` 为当前用户可见的每个 FE、External Catalog、元数据缓存引擎和缓存模块显示一行数据。除缓存配置和 Caffeine 运行指标外，该表还显示 Doris 4.1.4 引入的保留内存上限、估算使用量、淘汰内存和准入拒绝信息。 ## 所属数据库 - `information_schema` - ## 表信息 -该表中一行表示“某个 FE 上、某个 external catalog 的一个 cache entry”的统计快照。 - | 列名 | 类型 | 说明 | -| :----------- | :--- | :----------- | -| FE_HOST | text | 上报该统计的 FE 主机 | -| CATALOG_NAME | text | Catalog 名字 | -| ENGINE_NAME | text | Meta cache 引擎名，如 `hive`、`iceberg`、`paimon` | -| ENTRY_NAME | text | 引擎内部的 cache entry 名，如 `schema`、`file`、`manifest` | -| EFFECTIVE_ENABLED | boolean | 综合 `enable` / `ttl-second` / `capacity` 后，该缓存是否真正生效 | -| CONFIG_ENABLED | boolean | 配置中的原始 `enable` 值 | -| AUTO_REFRESH | boolean | 该 entry 是否启用异步 refresh-after-write | -| TTL_SECOND | bigint | TTL 秒数。`0` 表示关闭，`-1` 表示永不过期 | -| CAPACITY | bigint | 最大条目数 | -| ESTIMATED_SIZE | bigint | 当前缓存条目估计数 | -| REQUEST_COUNT | bigint | 总请求数 | -| HIT_COUNT | bigint | 命中次数 | -| MISS_COUNT | bigint | 未命中次数 | -| HIT_RATE | double | 命中率 | -| LOAD_SUCCESS_COUNT | bigint | 成功加载次数 | -| LOAD_FAILURE_COUNT | bigint | 失败加载次数 | -| TOTAL_LOAD_TIME_MS | bigint | 总加载耗时，单位毫秒 | -| AVG_LOAD_PENALTY_MS | double | 平均加载耗时，单位毫秒 | -| EVICTION_COUNT | bigint | 被驱逐条目数 | -| INVALIDATE_COUNT | bigint | 显式失效次数 | -| LAST_LOAD_SUCCESS_TIME | text | 最近一次成功加载时间 | -| LAST_LOAD_FAILURE_TIME | text | 最近一次失败加载时间 | -| LAST_ERROR | text | 最近一次加载失败错误信息 | +|---|---|---| +| `FE_HOST` | STRING | 上报该行数据的 FE 节点。 | +| `CATALOG_NAME` | STRING | External Catalog 名称。 | +| `ENGINE_NAME` | STRING | 元数据缓存引擎，例如 `hive`、`iceberg` 或 `paimon`。 | +| `ENTRY_NAME` | STRING | 引擎中的缓存模块名称。 | +| `EFFECTIVE_ENABLED` | BOOLEAN | 综合 enable、TTL、capacity 和 weight 配置后，该模块当前是否实际生效。 | +| `CONFIG_ENABLED` | BOOLEAN | 配置的 `enable` 值。 | +| `AUTO_REFRESH` | BOOLEAN | 是否启用受管理的自动刷新。 | +| `TTL_SECOND` | BIGINT | 过期时间，单位为秒。`-1` 表示永不过期，`0` 表示关闭模块。 | +| `CAPACITY` | BIGINT | 配置的条目数容量。启用 `max-weight` 后不再同时作为条目数上限，但 `0` 仍会关闭模块。 | +| `ESTIMATED_SIZE` | BIGINT | 缓存映射数量的近似值。 | +| `REQUEST_COUNT` | BIGINT | 缓存查询总次数。 | +| `HIT_COUNT` | BIGINT | 缓存命中次数。 | +| `MISS_COUNT` | BIGINT | 缓存未命中次数。 | +| `HIT_RATE` | DOUBLE | 缓存命中率，范围为 `0.0` 到 `1.0`。 | +| `LOAD_SUCCESS_COUNT` | BIGINT | 缓存加载成功次数。 | +| `LOAD_FAILURE_COUNT` | BIGINT | 缓存加载失败次数。 | +| `TOTAL_LOAD_TIME_MS` | BIGINT | 缓存加载总耗时，单位为毫秒。 | +| `AVG_LOAD_PENALTY_MS` | DOUBLE | 平均加载耗时，单位为毫秒。 | +| `EVICTION_COUNT` | BIGINT | Caffeine 和本地预算主动淘汰的次数。 | +| `INVALIDATE_COUNT` | BIGINT | 显式失效缓存的次数。 | +| `LAST_LOAD_SUCCESS_TIME` | STRING | 最近一次加载成功时间。 | +| `LAST_LOAD_FAILURE_TIME` | STRING | 最近一次加载失败时间。 | +| `LAST_ERROR` | STRING | 最近一次加载错误；没有记录时为空。 | +| `WEIGHT_BOUNDED` | BOOLEAN | 当前模块是否受到 FE、Catalog 或模块级内存上限约束。 | +| `MAX_WEIGHT` | BIGINT | 模块的有效内存上限，单位为字节。 | +| `ESTIMATED_WEIGHT` | BIGINT | 当前模块已预留的估算内存，单位为字节。 | +| `EVICTION_WEIGHT` | BIGINT | 自动淘汰和本地预算淘汰累计释放的估算字节数。 | +| `WEIGHT_REJECT_COUNT` | BIGINT | 因估算不完整或权重预算不足而拒绝缓存准入的次数。 | +| `CATALOG_MAX_WEIGHT` | BIGINT | Catalog 内存上限，单位为字节。 | +| `CATALOG_ESTIMATED_WEIGHT` | BIGINT | 当前 Catalog 中受管理模块已预留的估算内存。 | +| `GLOBAL_MAX_WEIGHT` | BIGINT | FE 外部元数据缓存总内存上限，单位为字节。 | +| `GLOBAL_ESTIMATED_WEIGHT` | BIGINT | 当前 FE 上受管理外部元数据缓存已预留的估算内存。 | +| `LAST_WEIGHT_REJECT_REASON` | STRING | 最近一次权重准入拒绝原因。 | +对于仍按条目数管理的模块，或者未配置的父级上限，不适用的 weight 数值列使用 `-1`。 ## 使用示例 +查看按内存管理的模块及其最近一次准入结果： + ```sql -SELECT catalog_name, engine_name, entry_name, - effective_enabled, ttl_second, capacity, - estimated_size, hit_rate, last_error +SELECT fe_host, catalog_name, engine_name, entry_name, + max_weight, estimated_weight, eviction_weight, + weight_reject_count, last_weight_reject_reason, + catalog_max_weight, catalog_estimated_weight, + global_max_weight, global_estimated_weight FROM information_schema.catalog_meta_cache_statistics -ORDER BY catalog_name, engine_name, entry_name; +WHERE weight_bounded = true +ORDER BY fe_host, catalog_name, engine_name, entry_name; ``` -常见用法： +查看指定 Catalog 的缓存效果和加载失败信息： -- 用 `ENGINE_NAME` + `ENTRY_NAME` 定位具体的逻辑缓存。 -- 用 `EFFECTIVE_ENABLED`、`TTL_SECOND`、`CAPACITY` 确认实际生效的缓存策略。 -- 用 `HIT_RATE`、`ESTIMATED_SIZE`、`LOAD_FAILURE_COUNT`、`LAST_ERROR` 排查缓存行为。 +```sql +SELECT engine_name, entry_name, effective_enabled, + estimated_size, request_count, hit_rate, + load_failure_count, last_error +FROM information_schema.catalog_meta_cache_statistics +WHERE catalog_name = 'iceberg_ctl' +ORDER BY engine_name, entry_name; +``` diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx index 3220ff68b5a5e..87ccbcaf0bf51 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx @@ -89,15 +89,16 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ### 缓存属性配置（4.1.x+） {#meta-cache-unified-model} -各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity}`。 +各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity,max-weight}`。`max-weight` 当前只适用于部分已接入估算器的模块，详见[外表元数据缓存内存管理](../external-meta-cache-memory-management.md)。 | 属性 | 示例 | 含义 | |---|---|---| | `enable` | `true/false` | 是否启用该缓存模块。 | | `ttl-second` | `600`、`0`、`-1` | `0` 表示关闭缓存（即刻生效，可用于查看最新数据）；`-1` 表示永不过期；其他正整数表示按访问时间计算 TTL（秒）。 | -| `capacity` | `10000` | 最大缓存条目数（按条目数量计）。`0` 表示关闭。 | +| `capacity` | `10000` | 按条目数计算的最大容量。启用 `max-weight` 后不再同时作为条目数上限，但 `0` 仍会关闭缓存。 | +| `max-weight` | `1GB` | 可选的估算保留内存上限。Hive 当前仅 `partition_values` 支持；`0` 会关闭该模块。 | -**生效逻辑说明：** 只有当 `enable=true` 且 `ttl-second != 0` 且 `capacity > 0` 时，该模块缓存才会生效。 +**生效逻辑说明：** 只有当 `enable=true`、`ttl-second != 0`、`capacity > 0`，并且 `max-weight` 未配置或不为 `0` 时，该模块缓存才会生效。 ### 缓存模块 {#meta-cache-unified-modules} @@ -136,7 +137,7 @@ Hive Catalog 包含以下缓存模块： * **性能优化**：对于元数据变动不频繁的场景，建议适当增大 `capacity` 和 `ttl-second` 以减少对 Hive Metastore 和文件系统的访问压力。 :::caution -**Hive Catalog 注意事项**：Hive 的 `meta.cache.hive.*` 属性修改**不支持热生效**。修改配置后，必须重建 Catalog 或重启 FE 节点才能应用新的缓存配置。 +**Hive Catalog 注意事项**：从 Doris 4.1.4 开始，Catalog 属性修改成功后会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问使用新的上下文和配置重建。已经开始的查询不受影响。 ::: ### 可观测性 {#meta-cache-unified-observability} diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hudi-catalog.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hudi-catalog.md index 11aa62f55092f..0422f4e048ed3 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hudi-catalog.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/hudi-catalog.md @@ -62,7 +62,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ### 缓存属性配置（4.1.x+） {#meta-cache-unified-model} -各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity}`。 +Hudi cache entry 使用配置键格式 `meta.cache.hudi..{enable,ttl-second,capacity}`。Hudi 模块当前仍按条目数管理，不接受模块级 `max-weight`，详见[外表元数据缓存内存管理](../external-meta-cache-memory-management.md)。 | 属性 | 示例 | 含义 | |---|---|---| @@ -102,7 +102,7 @@ Hudi Catalog 包含以下缓存模块： -- 关闭分区元数据缓存，以感知 Hudi 表的最新分区变动 ALTER CATALOG hudi_ctl SET PROPERTIES ("meta.cache.hudi.partition.ttl-second" = "0"); ``` -* **性能优化**：`ALTER CATALOG ... SET PROPERTIES` 的修改在 Hudi 中支持热生效（通过 HMS catalog 属性更新路径）。 +* **性能优化**：自 Doris 4.1.4 起，Catalog 属性修改成功后会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问使用新的上下文和配置重建，正在执行的查询不受影响。 ### 可观测性 {#meta-cache-unified-observability} diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx index 908c75302d30f..67e7ae8356ded 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx @@ -99,7 +99,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ## 元数据缓存 {#meta-cache} -为了提升访问外部数据源的性能，Apache Doris 会对 Iceberg 的元数据进行缓存。元数据包括表结构（Schema）、表对象、View 对象和 Manifest 详情等。 +为了提升访问外部数据源的性能，Apache Doris 会对 Iceberg 的元数据进行缓存。元数据包括表结构（Schema）、表对象、Snapshot、View 对象和 Manifest 详情等。 :::tip 对于 Doris 4.1.x 之前的版本，元数据缓存主要由 FE 配置项全局控制，详情请参阅[元数据缓存](../meta-cache.md)。 @@ -108,15 +108,16 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ### 缓存属性配置（4.1.x+） {#meta-cache-unified-model} -各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity}`。 +各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity,max-weight}`。内存上限只适用于已接入估算器的模块，详见[外表元数据缓存内存管理](../external-meta-cache-memory-management.md)。 | 属性 | 示例 | 含义 | |---|---|---| | `enable` | `true/false` | 是否启用该缓存模块。 | | `ttl-second` | `600`、`0`、`-1` | `0` 表示关闭缓存（即刻生效，可用于查看最新数据）；`-1` 表示永不过期；其他正整数表示按访问时间计算 TTL（秒）。 | -| `capacity` | `10000` | 最大缓存条目数（按条目数量计）。`0` 表示关闭。 | +| `capacity` | `10000` | 按条目数计算的最大容量。启用 `max-weight` 后不再同时作为条目数上限，但 `0` 仍会关闭缓存。 | +| `max-weight` | `1GB` | 可选的估算保留内存上限。Iceberg 的 `table`、`snapshot`、`manifest` 支持；`0` 会关闭对应模块。 | -**生效逻辑说明：** 只有当 `enable=true` 且 `ttl-second != 0` 且 `capacity > 0` 时，该模块缓存才会生效。 +**生效逻辑说明：** 只有当 `enable=true`、`ttl-second != 0`、`capacity > 0`，并且 `max-weight` 未配置或不为 `0` 时，该模块缓存才会生效。 ### 缓存模块 {#meta-cache-unified-modules} @@ -126,6 +127,7 @@ Iceberg Catalog 包含以下缓存模块： |---|---|---| | `schema` | `meta.cache.iceberg.schema.` | 缓存表结构。影响：列新增、删除、类型变更在 Doris 中的可见性。若关闭，每次查询都会拉取最新 Schema。 | | `table` | `meta.cache.iceberg.table.` | 缓存 Iceberg 表元数据对象。影响：最新 Snapshot、Partition Spec、Sort Order、表属性等表级元数据在 Doris 中的可见性；若关闭，每次规划都会重新加载表元数据。 | +| `snapshot` | `meta.cache.iceberg.snapshot.` | 缓存 Snapshot 元数据及其保留的表元数据代际，减少重复加载 Snapshot 和分区投影的开销；支持 `max-weight`。 | | `view` | `meta.cache.iceberg.view.` | 缓存 Iceberg View 元数据对象。影响：View 定义、Schema、属性变更在 Doris 中的可见性。 | | `manifest` | `meta.cache.iceberg.manifest.` | 缓存 Manifest 详情。主要影响查询规划时重复读取 Manifest 文件的开销，通常不直接决定表或 Snapshot 是否可见。注意：该模块默认关闭，需手动启用。 | @@ -156,7 +158,7 @@ Iceberg Catalog 包含以下缓存模块： "meta.cache.iceberg.manifest.ttl-second" = "600" ); ``` - * `ALTER CATALOG ... SET PROPERTIES` 的修改在 Iceberg Catalog 中支持热生效。 + * 自 Doris 4.1.4 起，Catalog 属性修改成功后会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问使用新的上下文和配置重建，正在执行的查询不受影响。 ### 可观测性 {#meta-cache-unified-observability} @@ -181,7 +183,7 @@ ORDER BY entry_name; | 3.0 | 1.6.1 | | 3.1 | 1.9.1 | | 4.0 | 1.9.1 | -| 4.1 | 1.10.1 | +| 4.1（4.1.4+） | 1.10.1 | ## 支持的 Iceberg 格式 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md index 6e8656b9d6259..42ffd7a3fdf97 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md @@ -122,7 +122,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ### 缓存属性配置（4.1.x+） {#meta-cache-unified-model} -各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity}`。 +MaxCompute cache entry 使用配置键格式 `meta.cache.maxcompute..{enable,ttl-second,capacity}`。MaxCompute 模块当前仍按条目数管理，不接受模块级 `max-weight`，详见[外表元数据缓存内存管理](../external-meta-cache-memory-management.md)。 | 属性 | 示例 | 含义 | |---|---|---| @@ -159,7 +159,7 @@ MaxCompute Catalog 包含以下缓存模块： -- 关闭分区值缓存，以感知 MaxCompute 表的最新分区 ALTER CATALOG mc_ctl SET PROPERTIES ("meta.cache.maxcompute.partition_values.ttl-second" = "0"); ``` -* **注意**：`meta.cache.maxcompute.*` 目前没有专门的热生效 hook。修改配置后，建议重建 Catalog 或重启 FE 以确保生效。 +* **注意**：自 Doris 4.1.4 起，Catalog 属性修改成功后会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问使用新的上下文和配置重建，正在执行的查询不受影响。 ### 可观测性 {#meta-cache-unified-observability} diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx index 70e4b9d100548..cb0bad0b090d0 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx @@ -222,7 +222,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ## 元数据缓存 {#meta-cache} -为了提升访问外部数据源的性能，Apache Doris 会对 Paimon 的元数据进行缓存。元数据包括表结构（Schema）和表对象等。 +为了提升访问外部数据源的性能，Apache Doris 会对 Paimon 的元数据进行缓存。元数据包括表结构（Schema）、表对象、Snapshot 和分区投影等。 :::tip 对于 Doris 4.1.x 之前的版本，元数据缓存主要由 FE 配置项全局控制，详情请参阅[元数据缓存](../meta-cache.md)。 @@ -231,15 +231,16 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ### 缓存属性配置（4.1.x+） {#meta-cache-unified-model} -各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity}`。 +各引擎 cache entry 使用统一的配置键格式：`meta.cache...{enable,ttl-second,capacity,max-weight}`。内存上限适用于已接入估算器的 `table` 和 `snapshot` 模块，详见[外表元数据缓存内存管理](../external-meta-cache-memory-management.md)。 | 属性 | 示例 | 含义 | |---|---|---| | `enable` | `true/false` | 是否启用该缓存模块。 | | `ttl-second` | `600`、`0`、`-1` | `0` 表示关闭缓存（即刻生效，可用于查看最新数据）；`-1` 表示永不过期；其他正整数表示按访问时间计算 TTL（秒）。 | -| `capacity` | `10000` | 最大缓存条目数（按条目数量计）。`0` 表示关闭。 | +| `capacity` | `10000` | 按条目数计算的最大容量。启用 `max-weight` 后不再同时作为条目数上限，但 `0` 仍会关闭缓存。 | +| `max-weight` | `1GB` | 可选的估算保留内存上限。Paimon 的 `table` 和 `snapshot` 支持；`0` 会关闭对应模块。 | -**生效逻辑说明：** 只有当 `enable=true` 且 `ttl-second != 0` 且 `capacity > 0` 时，该模块缓存才会生效。 +**生效逻辑说明：** 只有当 `enable=true`、`ttl-second != 0`、`capacity > 0`，并且 `max-weight` 未配置或不为 `0` 时，该模块缓存才会生效。 ### 缓存模块 {#meta-cache-unified-modules} @@ -249,6 +250,7 @@ Paimon Catalog 包含以下缓存模块： |---|---|---| | `schema` | `meta.cache.paimon.schema.` | 缓存表结构。影响：列新增、删除、类型变更在 Doris 中的可见性。若关闭，每次查询都会拉取最新 Schema。 | | `table` | `meta.cache.paimon.table.` | 缓存 Paimon 表元数据对象。影响：最新 Snapshot、Schema 演进、分支/标签引用等表级元数据在 Doris 中的可见性，同时减少查询规划时的元数据加载开销。 | +| `snapshot` | `meta.cache.paimon.snapshot.` | 缓存 Snapshot 元数据、Schema 代际和分区投影；支持 `max-weight`。 | ### 旧参数映射与转换 {#meta-cache-mapping} @@ -268,7 +270,7 @@ Paimon Catalog 包含以下缓存模块： -- 关闭表对象缓存，以感知 Paimon 表的最新快照 ALTER CATALOG paimon_ctl SET PROPERTIES ("meta.cache.paimon.table.ttl-second" = "0"); ``` -* **性能优化**：`ALTER CATALOG ... SET PROPERTIES` 的修改在 Paimon Catalog 中支持热生效。 +* **性能优化**：自 Doris 4.1.4 起，Catalog 属性修改成功后会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问使用新的上下文和配置重建，正在执行的查询不受影响。启用 Doris 内存治理后，默认关闭 Paimon SDK 自带的 `CachingCatalog`；显式配置 `paimon.cache-enabled` 时以用户配置为准。 ### 可观测性 {#meta-cache-unified-observability} @@ -287,7 +289,7 @@ ORDER BY entry_name; ### 支持的 Paimon 版本 -当前依赖的 Paimon 版本为 1.0.0。 +Doris 4.1.4 当前依赖的 Paimon 版本为 1.4.2。 ### 支持的 Paimon 格式 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/external-meta-cache-memory-management.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/external-meta-cache-memory-management.md new file mode 100644 index 0000000000000..e0e77009691f5 --- /dev/null +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/external-meta-cache-memory-management.md @@ -0,0 +1,295 @@ +--- +{ + "title": "外部元数据缓存内存管理", + "language": "zh-CN", + "description": "介绍 Doris 4.1.4 中外部元数据缓存的 FE、Catalog 和缓存模块三级内存限制，以及 Hive、Iceberg 和 Paimon 的配置方法。" +} +--- + +从 Doris 4.1.4 开始，可以按估算内存大小限制部分外部元数据缓存，避免单个大表、单个 Catalog 或多个 Catalog 的元数据缓存持续占用 FE Heap。 + +该功能适合以下场景： + +- Hive 表包含大量分区，分区裁剪结构占用较多 FE 内存； +- Iceberg 表的表元数据、Snapshot 或 Manifest 较大； +- Paimon 表元数据代际或 Snapshot 包含大量保留元数据和分区投影； +- 一个 FE 同时服务多个 Catalog，需要避免单个 Catalog 占用过多缓存内存。 + +:::caution +内存限制针对本文列出的、支持内存估算的缓存模块，不是 FE Heap 的硬限制。Catalog 客户端、线程池等共享运行时基础设施不会全部归属于单个缓存项；但缓存项实际持有的认证上下文、Iceberg FileIO 配置和凭证等有界 owner payload 会按保守值计入。尚未接入内存估算的元数据缓存不计入该配额。 +::: + +## 快速开始 + +使用该功能前，需要具备以下条件： + +- 已创建 Hive、Iceberg 或 Paimon Catalog； +- 具有修改 FE 配置和对应 Catalog 属性的权限； +- 所有 FE 节点都使用 Doris 4.1.4 或更高版本。 + +### 1. 配置 FE 总上限 + +在每个 FE 节点的 `fe.conf` 中设置： + +```properties +external_meta_cache_max_weight = 10% +``` + +该配置表示：当前 FE 上所有已接入内存管理的外部元数据缓存，合计最多使用当前 JVM 最大 Heap 的 10%。配置修改后需要重启 FE。 + +如果不希望按 Heap 比例配置，也可以使用固定大小： + +```properties +external_meta_cache_max_weight = 8GB +``` + +### 2. 限制单个 Catalog + +下面的示例将 `iceberg_ctl` 中受管理的元数据缓存总量限制为 4 GB： + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB" +); +``` + +修改成功后，Doris 会清理该 Catalog 已初始化的相关缓存，并在下一次访问时使用新配置重新创建缓存。 + +:::tip +即使没有配置 FE 总上限，也可以单独配置 `meta.cache.max-weight`。此时只限制该 Catalog，不限制当前 FE 上所有 Catalog 的合计值。 +::: + +### 3. 限制具体缓存模块 + +下面的示例同时限制 Iceberg Catalog 总量以及各缓存模块： + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB", + "meta.cache.iceberg.table.max-weight" = "1GB", + "meta.cache.iceberg.snapshot.max-weight" = "2GB", + "meta.cache.iceberg.manifest.enable" = "true", + "meta.cache.iceberg.manifest.max-weight" = "1GB" +); +``` + +配置后： + +- Iceberg `table` 缓存最多占用 1 GB； +- Iceberg `snapshot` 缓存最多占用 2 GB； +- Iceberg `manifest` 缓存最多占用 1 GB； +- 三者合计不能超过 Catalog 的 4 GB 上限； +- 该 Catalog 的内存仍受 FE 总上限约束。 + +## 配额层级 + +外部元数据缓存支持三级内存限制： + +| 层级 | 配置位置 | 配置项 | 作用范围 | +|---|---|---|---| +| FE | `fe.conf` | `external_meta_cache_max_weight` | 当前 FE 上所有受管理的外部元数据缓存 | +| Catalog | Catalog 属性 | `meta.cache.max-weight` | 当前 Catalog 下所有受管理的缓存模块 | +| 缓存模块 | Catalog 属性 | `meta.cache...max-weight` | 指定引擎的一个缓存模块 | + +最终有效上限取所有已配置上限中的最小值。不同配置组合的行为如下： + +| FE 上限 | Catalog 上限 | 模块上限 | 实际行为 | +|---|---|---|---| +| 未配置 | 未配置 | 未配置 | 不启用按内存计费，继续使用原有的条目数和 TTL 策略 | +| 已配置 | 未配置 | 未配置 | 所有受管理模块共享 FE 总配额 | +| 未配置 | 已配置 | 未配置 | 当前 Catalog 内的受管理模块共享 Catalog 配额 | +| 未配置 | 未配置 | 已配置 | 只限制指定缓存模块，不限制多个模块或 Catalog 的合计值 | +| 已配置 | 已配置 | 已配置 | 同时满足三级限制，取最严格的上限 | + +配置必须满足以下层级关系： + +- Catalog 上限不能大于 FE 总上限； +- 模块上限不能大于已配置的直接父级上限； +- 在不同 FE Heap 大小不一致的集群中，从节点会按本地 FE 总上限进一步收紧配额。 + +如果配置违反层级关系，`CREATE CATALOG` 或 `ALTER CATALOG` 会失败，而不会静默忽略配置。 + +## 配置值格式 + +固定大小支持以下单位，单位不区分大小写，按 1024 进制换算： + +```text +B, KB, MB, GB, TB, PB +``` + +例如：`512MB`、`4GB`。 + +固定大小只支持整数；只有百分比支持小数，例如 `12.5%`。 + +只有 FE 配置 `external_meta_cache_max_weight` 支持百分比，例如 `10%`、`12.5%`。百分比以每个 FE 自身的 JVM 最大 Heap 为基准计算，因此不同 Heap 大小的 FE 会得到不同的字节上限。 + +各层级对 `0` 的处理不同： + +| 配置项 | `0` 的含义 | +|---|---| +| `external_meta_cache_max_weight` | 关闭 FE 总配额，但不会关闭缓存 | +| `meta.cache.max-weight` | 不允许设置为 `0` | +| `meta.cache...max-weight` | 关闭该缓存模块，而不是取消模块级覆盖 | + +`0%` 不是有效的 FE 配置。需要关闭 FE 总配额时，请使用不带百分号的 `0`。 + +## 支持按内存限制的缓存模块 + +当前版本只允许为下列缓存模块设置 `max-weight`： + +| Catalog/引擎 | 缓存模块 | 配置项 | 缓存内容 | +|---|---|---|---| +| Hive | `partition_values` | `meta.cache.hive.partition_values.max-weight` | 分区名称、分区值、分区裁剪索引和排序范围 | +| Iceberg | `table` | `meta.cache.iceberg.table.max-weight` | Iceberg 表元数据及当前元数据代际 | +| Iceberg | `snapshot` | `meta.cache.iceberg.snapshot.max-weight` | Snapshot、分区投影和与其绑定的表元数据代际 | +| Iceberg | `manifest` | `meta.cache.iceberg.manifest.max-weight` | 解析后的 DataFile 和 DeleteFile 列表；该模块默认关闭，使用前还需设置 `enable=true` | +| Paimon | `table` | `meta.cache.paimon.table.max-weight` | 保留的 Paimon 表元数据及其元数据代际 | +| Paimon | `snapshot` | `meta.cache.paimon.snapshot.max-weight` | Snapshot、Schema 代际和分区投影 | + +以下常见模块目前仍按条目数管理，不能配置 `max-weight`： + +- Hive `schema`、`partition`、`file`； +- Iceberg `schema`、`view`； +- Paimon `schema`； +- Hudi、MaxCompute 和 Doris Catalog 的现有缓存模块。 + +为不支持的模块设置 `max-weight` 会导致 Catalog 创建或修改失败。例如，下面的配置无效： + +```sql +ALTER CATALOG hive_ctl SET PROPERTIES ( + "meta.cache.hive.file.max-weight" = "1GB" +); +``` + +:::note +HMS Catalog 可以同时路由 Hive、Hudi 和 Iceberg 元数据缓存。如果 HMS 中包含 Iceberg 表，可以在同一个 HMS Catalog 上使用 `meta.cache.iceberg.table.max-weight`、`meta.cache.iceberg.snapshot.max-weight` 和 `meta.cache.iceberg.manifest.max-weight`。 +::: + +## 配置示例 + +### 只使用 Catalog 上限 + +如果 FE 没有统一总上限，可以分别隔离不同 Catalog： + +```sql +ALTER CATALOG hive_prod SET PROPERTIES ( + "meta.cache.max-weight" = "6GB" +); + +ALTER CATALOG iceberg_ad_hoc SET PROPERTIES ( + "meta.cache.max-weight" = "2GB" +); +``` + +此时两个 Catalog 分别受 6 GB 和 2 GB 限制，但当前 FE 上所有 Catalog 的合计值没有统一上限。 + +### 限制 Hive 大分区表 + +```sql +ALTER CATALOG hive_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB", + "meta.cache.hive.partition_values.max-weight" = "3GB" +); +``` + +该配置主要限制大分区表生成的分区裁剪结构。Hive 文件列表缓存当前不计入此配额。 + +### 限制 Paimon 表元数据代际和 Snapshot + +```sql +ALTER CATALOG paimon_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "2GB", + "meta.cache.paimon.table.max-weight" = "512MB", + "meta.cache.paimon.snapshot.max-weight" = "1536MB" +); +``` + +Paimon 的 `table` 和 `snapshot` 模块都支持独立的内存上限。该示例为表元数据代际最多分配 512 MB，为 Snapshot 投影最多分配 1536 MB，二者合计仍受 2 GB Catalog 上限约束。 + +### 与现有缓存属性的兼容关系 + +为兼容已有 Catalog 配置，Iceberg 和 Paimon 的 `table.enable`、`table.ttl-second`、`table.capacity` 会在未显式配置对应 `snapshot` 属性时同时作为 Snapshot 缓存的默认值。例如： + +```sql +ALTER CATALOG paimon_ctl SET PROPERTIES ( + "meta.cache.paimon.table.ttl-second" = "600", + "meta.cache.paimon.snapshot.max-weight" = "1536MB" +); +``` + +其中 `table.ttl-second` 同时控制 Paimon 表对象和 Snapshot 缓存的 TTL，而 `snapshot.max-weight` 只限制 Snapshot 缓存。 + +`max-weight` 不参与上述兼容映射。必须使用实际支持内存估算的模块名： + +- Iceberg 表对象使用 `meta.cache.iceberg.table.max-weight`； +- Iceberg Snapshot 使用 `meta.cache.iceberg.snapshot.max-weight`； +- Paimon 表元数据代际使用 `meta.cache.paimon.table.max-weight`； +- Paimon Snapshot 使用 `meta.cache.paimon.snapshot.max-weight`。 + +### 同时使用内存上限和 TTL + +内存上限可以和 `enable`、`ttl-second`、`capacity` 一起配置： + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.iceberg.snapshot.enable" = "true", + "meta.cache.iceberg.snapshot.ttl-second" = "1800", + "meta.cache.iceberg.snapshot.capacity" = "1000", + "meta.cache.iceberg.snapshot.max-weight" = "2GB" +); +``` + +启用 `max-weight` 后，缓存的淘汰上限按内存权重执行，不再同时使用 `capacity` 作为最大条目数。但 `capacity=0` 仍会关闭缓存；因此使用内存限制时，应保留一个大于 0 的 `capacity`。 + +## 达到内存上限时的行为 + +一次缓存未命中的处理流程如下： + +1. Doris 从外部数据源加载元数据； +2. 对即将缓存的对象进行冻结或物化，计算其保留内存大小； +3. 同时检查 FE、Catalog 和缓存模块三级配额； +4. 配额不足时，优先淘汰同一缓存模块中的冷数据并重试；如果本地无法满足预留，则异步请求回收同 Catalog 的兄弟模块（FE 总配额不足时也会尝试其他 Catalog）； +5. 仍无法满足配额时，不将新对象写入缓存，但把已经加载的对象返回给当前请求。 + +因此，单个新对象大于可用配额时，正常查询不会因为缓存配额不足而失败。不过该对象不会被缓存，后续访问可能再次从外部数据源加载，导致查询规划时间增加。 + +如果对象无法完成可靠估算，例如保留的表类型不受支持、Iceberg 元数据仍处于懒加载状态、准备过程失败或估算工作量超过安全预算，Doris 同样会放弃缓存，而不是使用不完整结果低估内存。估算器不做通用反射对象图遍历，也不对容器采样。Iceberg 只使用一个 fail-closed 私有字段探针确认 Snapshot 元数据已经加载，从而避免估算过程触发 IO。 + +:::note +本地淘汰是同步的，peer reclaim 是异步且尽力而为的。因此当前这次未命中仍可能不写入缓存，而兄弟模块的回收会影响后续访问。若不希望某个模块长期占用共享的 FE 或 Catalog 配额，应为热点模块分别设置模块级上限。 +::: + +定时刷新得到的新值如果估算不完整或因配额不足无法准入，Doris 会继续保留之前已知可用的缓存代际，不发布被拒绝的刷新结果。 + +:::caution +配额检查发生在对象加载和准备完成后的缓存准入阶段，不会在访问外部数据源前预留 Heap。因此，远端加载失败，或者单个超大对象在构建完成前已经耗尽 FE Heap，仍可能使当前请求失败。该功能限制的是成功构建后可保留在缓存中的内存，不能作为单次元数据加载的 OOM 防护。 +::: + +## 配置生效与缓存刷新 + +- 修改 `external_meta_cache_max_weight` 后，需要重启对应 FE； +- 修改 Catalog 属性（包括 `meta.cache.max-weight` 或 `meta.cache...*`）会重置 Catalog 执行上下文，并清理该 Catalog 路由到的所有元数据缓存引擎中已初始化的条目；下一次访问会使用新的上下文和配置重建； +- 缓存重建期间，已经开始的查询可以继续使用已加载对象；后续访问按新配置重新加载和缓存。 + +内存上限只控制对象是否可以保留在缓存中，不改变外部元数据本身，也不代替 `REFRESH CATALOG`、TTL 或元数据事件同步。 + +## 注意事项 + +- `max-weight` 是估算的保留缓存准入预算。类型专用公式使用离线校准后向上取整的结构常量，并叠加 loader 阶段统计的 payload；它不模拟当前 JVM 或第三方 SDK 的精确对象布局，也不等同于操作系统 RSS，不能代替 FE Heap 和 GC 监控； +- 共享运行时基础设施不会全部归属于单个缓存项，但缓存项实际持有的认证上下文、FileIO 配置、存储凭证和 Transform payload 会按有界保守值计入； +- 按权重管理的缓存使用 soft value。Reservation 只保留 key、generation 和 weight，不强引用缓存 value。Heap 压力下 JVM 可能在 TTL 到期或配额用满前回收 value；Doris 会释放对应 reservation，后续访问重新加载； +- 大对象只在加载、刷新或替换时估算。完成的权重随发布代际保留，缓存命中不会重新扫描对象内容； +- Iceberg 首次缓存表或 Snapshot 时，可能需要提前物化当前 Snapshot 的 Manifest 列表，因此第一次加载的耗时可能增加； +- 新提交的 DDL 属性会严格拒绝拼写错误的引擎名、模块名和参数名；历史持久化属性在运行时无法解析时会告警并忽略，使升级后的 Catalog 仍可初始化； +- Iceberg Catalog 启用 Doris 内存治理后，不再自动开启 Iceberg SDK 的 Manifest 内容缓存，因为该缓存位于 Doris 配额之外；显式配置 `io.manifest.cache-enabled` 时以用户配置为准； +- Paimon Catalog 启用 Doris 内存治理后，默认关闭 Paimon SDK 自带的 `CachingCatalog`，避免出现第二份无界元数据缓存；显式配置 `paimon.cache-enabled` 时以用户配置为准； +- 升级 Iceberg 或 Paimon SDK 后，应使用与当前 Doris 版本匹配的依赖，不建议单独替换 FE 中的 SDK JAR。 + +## 最佳实践 + +1. 优先配置 FE 总上限，防止多个 Catalog 的缓存合计失控； +2. 对共享 FE 上的大型生产 Catalog 配置 Catalog 上限，实现租户或工作负载隔离； +3. 只有在某个模块明显占用较大时，再增加模块级上限，避免为每个缓存模块都配置参数； +4. 为查询规划和其他 FE 缓存保留足够 Heap，不要把大部分 JVM Heap 都分配给外部元数据缓存； +5. 如果配额不足导致同一对象频繁重新加载，可适当提高上限，或降低其他模块的上限； +6. 对异构 FE 集群，使用百分比可以按各节点 Heap 自动缩放；需要所有节点保持相同字节上限时，使用固定大小。 diff --git a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/meta-cache.md b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/meta-cache.md index cf6145a5925a9..aa19f5d4b99e7 100644 --- a/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/meta-cache.md +++ b/i18n/zh-CN/docusaurus-plugin-content-docs/version-4.x/lakehouse/meta-cache.md @@ -20,6 +20,8 @@ 关于**数据缓存**，可参阅[数据缓存文档](./data-cache.md)。 +关于 Doris 4.1.4 外部元数据缓存的内存限制，请参阅[外部元数据缓存内存管理](./external-meta-cache-memory-management.md)。 + ## 缓存策略 大多数缓存都有如下三个策略指标： diff --git a/versioned_docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md b/versioned_docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md index 69d9388eb079b..6eae7b7e9ea8f 100644 --- a/versioned_docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md +++ b/versioned_docs/version-4.x/admin-manual/system-tables/information_schema/catalog_meta_cache_statistics.md @@ -2,64 +2,80 @@ { "title": "catalog_meta_cache_statistics", "language": "en", - "description": "View the metadata cache information of the External Catalog in the currently connected FE." + "description": "View External Catalog metadata-cache configuration, activity, and memory-governance statistics on FE nodes." } --- ## Overview -View the metadata cache information of the External Catalog in the currently connected FE. +`catalog_meta_cache_statistics` shows one row for each FE, External Catalog, metadata-cache engine, and cache entry visible to the current user. It includes cache configuration and Caffeine activity, together with retained-memory limits, estimated usage, eviction weight, and admission rejections introduced in Doris 4.1.4. ## Database - `information_schema` - ## Table Information -One row represents one cache entry on one FE for one external catalog. - -| Column Name | Type | Description | -| ------------ | ---- | ----------- | -| FE_HOST | text | FE host that reports the stats | -| CATALOG_NAME | text | Catalog name | -| ENGINE_NAME | text | Meta cache engine name, such as `hive`, `iceberg`, `paimon` | -| ENTRY_NAME | text | Cache entry name inside the engine, such as `schema`, `file`, `manifest` | -| EFFECTIVE_ENABLED | boolean | Whether the cache is effectively enabled after evaluating `enable` / `ttl-second` / `capacity` | -| CONFIG_ENABLED | boolean | Raw `enable` flag from the cache config | -| AUTO_REFRESH | boolean | Whether async refresh-after-write is enabled for this entry | -| TTL_SECOND | bigint | TTL in seconds. `0` means disabled; `-1` means no expiration | -| CAPACITY | bigint | Max entry count | -| ESTIMATED_SIZE | bigint | Estimated current cache size | -| REQUEST_COUNT | bigint | Total requests | -| HIT_COUNT | bigint | Cache hits | -| MISS_COUNT | bigint | Cache misses | -| HIT_RATE | double | Hit rate | -| LOAD_SUCCESS_COUNT | bigint | Successful loads | -| LOAD_FAILURE_COUNT | bigint | Failed loads | -| TOTAL_LOAD_TIME_MS | bigint | Total load time in milliseconds | -| AVG_LOAD_PENALTY_MS | double | Average load time in milliseconds | -| EVICTION_COUNT | bigint | Evicted entries | -| INVALIDATE_COUNT | bigint | Explicit invalidations | -| LAST_LOAD_SUCCESS_TIME | text | Last successful load time | -| LAST_LOAD_FAILURE_TIME | text | Last failed load time | -| LAST_ERROR | text | Latest load error message | +| Column | Type | Description | +|---|---|---| +| `FE_HOST` | STRING | FE node that reports the row. | +| `CATALOG_NAME` | STRING | External Catalog name. | +| `ENGINE_NAME` | STRING | Metadata-cache engine, such as `hive`, `iceberg`, or `paimon`. | +| `ENTRY_NAME` | STRING | Cache entry within the engine. | +| `EFFECTIVE_ENABLED` | BOOLEAN | Whether the entry is currently effective after evaluating enable, TTL, capacity, and weight settings. | +| `CONFIG_ENABLED` | BOOLEAN | Configured `enable` value. | +| `AUTO_REFRESH` | BOOLEAN | Whether managed automatic refresh is enabled. | +| `TTL_SECOND` | BIGINT | Expiration time in seconds. `-1` means no expiration and `0` disables the entry. | +| `CAPACITY` | BIGINT | Configured count capacity. With `max-weight`, it is not a simultaneous count limit, but `0` still disables the entry. | +| `ESTIMATED_SIZE` | BIGINT | Approximate number of cached mappings. | +| `REQUEST_COUNT` | BIGINT | Total cache lookup requests. | +| `HIT_COUNT` | BIGINT | Cache hits. | +| `MISS_COUNT` | BIGINT | Cache misses. | +| `HIT_RATE` | DOUBLE | Cache hit rate from `0.0` to `1.0`. | +| `LOAD_SUCCESS_COUNT` | BIGINT | Successful cache loads. | +| `LOAD_FAILURE_COUNT` | BIGINT | Failed cache loads. | +| `TOTAL_LOAD_TIME_MS` | BIGINT | Total cache load time in milliseconds. | +| `AVG_LOAD_PENALTY_MS` | DOUBLE | Average load time in milliseconds. | +| `EVICTION_COUNT` | BIGINT | Number of Caffeine and explicit local-budget evictions. | +| `INVALIDATE_COUNT` | BIGINT | Number of explicit invalidations. | +| `LAST_LOAD_SUCCESS_TIME` | STRING | Time of the most recent successful load. | +| `LAST_LOAD_FAILURE_TIME` | STRING | Time of the most recent failed load. | +| `LAST_ERROR` | STRING | Most recent load error; empty when none is recorded. | +| `WEIGHT_BOUNDED` | BOOLEAN | Whether this entry is governed by an applicable FE, Catalog, or entry memory limit. | +| `MAX_WEIGHT` | BIGINT | Effective entry memory limit in bytes. | +| `ESTIMATED_WEIGHT` | BIGINT | Current reserved estimated weight of this entry in bytes. | +| `EVICTION_WEIGHT` | BIGINT | Cumulative estimated bytes released by automatic and local-budget eviction. | +| `WEIGHT_REJECT_COUNT` | BIGINT | Number of cache admissions rejected by incomplete estimation or insufficient weight budget. | +| `CATALOG_MAX_WEIGHT` | BIGINT | Catalog memory limit in bytes. | +| `CATALOG_ESTIMATED_WEIGHT` | BIGINT | Current reserved estimated weight across managed entries in this Catalog. | +| `GLOBAL_MAX_WEIGHT` | BIGINT | FE-wide external metadata-cache memory limit in bytes. | +| `GLOBAL_ESTIMATED_WEIGHT` | BIGINT | Current reserved estimated weight across managed external metadata caches on the FE. | +| `LAST_WEIGHT_REJECT_REASON` | STRING | Most recent weight-admission rejection reason. | +For count-bounded entries or an unconfigured parent limit, weight-related numeric columns use `-1` when the value is not applicable. ## Usage Example +Inspect memory-governed entries and their most recent admission result: + ```sql -SELECT catalog_name, engine_name, entry_name, - effective_enabled, ttl_second, capacity, - estimated_size, hit_rate, last_error +SELECT fe_host, catalog_name, engine_name, entry_name, + max_weight, estimated_weight, eviction_weight, + weight_reject_count, last_weight_reject_reason, + catalog_max_weight, catalog_estimated_weight, + global_max_weight, global_estimated_weight FROM information_schema.catalog_meta_cache_statistics -ORDER BY catalog_name, engine_name, entry_name; +WHERE weight_bounded = true +ORDER BY fe_host, catalog_name, engine_name, entry_name; ``` -Typical usage: - -- Use `ENGINE_NAME` + `ENTRY_NAME` to identify one logical cache entry. -- Use `EFFECTIVE_ENABLED`, `TTL_SECOND`, and `CAPACITY` to confirm the applied cache policy. -- Use `HIT_RATE`, `ESTIMATED_SIZE`, `LOAD_FAILURE_COUNT`, and `LAST_ERROR` to diagnose behavior. +Inspect cache effectiveness and load failures for one Catalog: +```sql +SELECT engine_name, entry_name, effective_enabled, + estimated_size, request_count, hit_rate, + load_failure_count, last_error +FROM information_schema.catalog_meta_cache_statistics +WHERE catalog_name = 'iceberg_ctl' +ORDER BY engine_name, entry_name; +``` diff --git a/versioned_docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx b/versioned_docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx index 77f4772f43751..f94485279fd28 100644 --- a/versioned_docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx +++ b/versioned_docs/version-4.x/lakehouse/catalogs/hive-catalog.mdx @@ -87,15 +87,16 @@ Starting from Doris 4.1.x, Hive Catalog's external metadata cache is configured ### Cache Property Configuration (4.1.x+) {#meta-cache-unified-model} -Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity}`. +Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity,max-weight}`. `max-weight` is currently supported only by selected estimator-backed entries; see [External Metadata Cache Memory Management](../external-meta-cache-memory-management.md). | Property | Example | Meaning | |---|---|---| | `enable` | `true/false` | Whether to enable this cache module. | | `ttl-second` | `600`, `0`, `-1` | `0` means disable cache (takes effect immediately, can be used to see the latest data); `-1` means never expire; other positive integers mean TTL in seconds based on access time. | -| `capacity` | `10000` | Maximum number of cache entries (by count). `0` means disable. | +| `capacity` | `10000` | Maximum number of cache entries by count. When `max-weight` is active, it is not a simultaneous count limit, but `0` still disables the cache. | +| `max-weight` | `1GB` | Optional estimated retained-memory limit. Currently supported by Hive `partition_values`; `0` disables that entry. | -**Effective Logic:** The module cache only takes effect when `enable=true`, `ttl-second != 0`, and `capacity > 0`. +**Effective Logic:** The module cache takes effect when `enable=true`, `ttl-second != 0`, `capacity > 0`, and `max-weight` is either absent or not `0`. ### Cache Modules {#meta-cache-unified-modules} @@ -130,7 +131,7 @@ In version 4.1.x and later, unified keys are recommended. The following is the m * **Performance optimization**: For scenarios where metadata changes are infrequent, it is recommended to appropriately increase `capacity` and `ttl-second` to reduce access pressure on Hive Metastore and file systems. :::caution -**Hive Catalog Note**: Changes to `meta.cache.hive.*` properties **do not support hot-reload**. To ensure new configurations take effect, you must recreate the catalog or restart the FE node. +**Hive Catalog Note**: Starting from Doris 4.1.4, a successful Catalog property change resets the Catalog execution context and clears initialized entries for every metadata-cache engine routed by that Catalog. Doris recreates them with the new context and configuration on the next access. Queries already in progress are not affected. ::: ### Observability {#meta-cache-unified-observability} diff --git a/versioned_docs/version-4.x/lakehouse/catalogs/hudi-catalog.md b/versioned_docs/version-4.x/lakehouse/catalogs/hudi-catalog.md index 974f37847c2c7..04a6884930a16 100644 --- a/versioned_docs/version-4.x/lakehouse/catalogs/hudi-catalog.md +++ b/versioned_docs/version-4.x/lakehouse/catalogs/hudi-catalog.md @@ -62,7 +62,7 @@ Starting from Doris 4.1.x, Hudi-related external metadata cache is configured us ### Cache Property Configuration (4.1.x+) {#meta-cache-unified-model} -Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity}`. +Hudi cache entries use the configuration key format `meta.cache.hudi..{enable,ttl-second,capacity}`. Hudi entries are currently count-bounded and do not accept entry-level `max-weight`; see [External Metadata Cache Memory Management](../external-meta-cache-memory-management.md). | Property | Example | Meaning | |---|---|---| @@ -98,7 +98,7 @@ In version 4.1.x and later, unified keys are recommended. The following is the m -- Disable partition metadata cache to detect the latest partition changes in Hudi tables ALTER CATALOG hudi_ctl SET PROPERTIES ("meta.cache.hudi.partition.ttl-second" = "0"); ``` -* **Performance optimization**: Changes via `ALTER CATALOG ... SET PROPERTIES` support hot-reload in Hudi (via the HMS catalog property update path). +* **Performance optimization**: Starting from Doris 4.1.4, a successful Catalog property change resets the Catalog execution context and clears initialized entries for every metadata-cache engine routed by that Catalog. Doris rebuilds them with the new context and configuration on the next access; in-flight queries are not affected. ### Observability {#meta-cache-unified-observability} diff --git a/versioned_docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx b/versioned_docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx index 5e0cb16d27519..c858baebfb856 100644 --- a/versioned_docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx +++ b/versioned_docs/version-4.x/lakehouse/catalogs/iceberg-catalog.mdx @@ -99,7 +99,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ## Metadata Cache {#meta-cache} -To improve the performance of accessing external data sources, Apache Doris caches Iceberg metadata. Metadata includes table structure (Schema), table objects, view objects, and manifest details. +To improve the performance of accessing external data sources, Apache Doris caches Iceberg metadata. Metadata includes table structure (Schema), table objects, snapshots, view objects, and manifest details. :::tip For versions before Doris 4.1.x, metadata caching is mainly controlled globally by FE configuration items. For details, see [Metadata Cache](../meta-cache.md). @@ -108,15 +108,16 @@ Starting from Doris 4.1.x, Iceberg Catalog's external metadata cache is configur ### Cache Property Configuration (4.1.x+) {#meta-cache-unified-model} -Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity}`. +Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity,max-weight}`. Memory limits apply only to estimator-backed entries; see [External Metadata Cache Memory Management](../external-meta-cache-memory-management.md). | Property | Example | Meaning | |---|---|---| | `enable` | `true/false` | Whether to enable this cache module. | | `ttl-second` | `600`, `0`, `-1` | `0` means disable cache (takes effect immediately, can be used to see the latest data); `-1` means never expire; other positive integers mean TTL in seconds based on access time. | -| `capacity` | `10000` | Maximum number of cache entries (by count). `0` means disable. | +| `capacity` | `10000` | Maximum number of cache entries by count. When `max-weight` is active, it is not a simultaneous count limit, but `0` still disables the cache. | +| `max-weight` | `1GB` | Optional estimated retained-memory limit. Supported by Iceberg `table`, `snapshot`, and `manifest`; `0` disables that entry. | -**Effective Logic:** The module cache only takes effect when `enable=true`, `ttl-second != 0`, and `capacity > 0`. +**Effective Logic:** The module cache takes effect when `enable=true`, `ttl-second != 0`, `capacity > 0`, and `max-weight` is either absent or not `0`. ### Cache Modules {#meta-cache-unified-modules} @@ -126,6 +127,7 @@ Iceberg Catalog includes the following cache modules: |---|---|---| | `schema` | `meta.cache.iceberg.schema.` | Caches table structure. Impact: Visibility of table column information. If disabled, the latest Schema is pulled for each query. | | `table` | `meta.cache.iceberg.table.` | Caches Iceberg table metadata objects. Impact: Reduces Catalog/Metastore round-trips. | +| `snapshot` | `meta.cache.iceberg.snapshot.` | Caches snapshot metadata and retained table generations. Impact: Reduces repeated snapshot and partition-projection loading. This entry supports `max-weight`. | | `view` | `meta.cache.iceberg.view.` | Caches Iceberg View metadata objects. | | `manifest` | `meta.cache.iceberg.manifest.` | Caches manifest details. Impact: Reduces repeated manifest access overhead. Note: This module is disabled by default and must be enabled manually. | @@ -152,7 +154,7 @@ In version 4.1.x and later, unified keys are recommended. The following is the m "meta.cache.iceberg.manifest.ttl-second" = "600" ); ``` - * Changes via `ALTER CATALOG ... SET PROPERTIES` support hot-reload in Iceberg Catalog. + * Starting from Doris 4.1.4, a successful Catalog property change resets the Catalog execution context and clears initialized entries for every metadata-cache engine routed by that Catalog. Doris rebuilds them with the new context and configuration on the next access; in-flight queries are not affected. ### Observability {#meta-cache-unified-observability} @@ -177,6 +179,7 @@ See the documentation for this system table: [catalog_meta_cache_statistics](../ | 3.0 | 1.6.1 | | 3.1 | 1.9.1 | | 4.0 | 1.9.1 | +| 4.1 (4.1.4+) | 1.10.1 | ### Supported Iceberg Formats diff --git a/versioned_docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md b/versioned_docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md index 0a50fa93003a3..b5aa57199a1ea 100644 --- a/versioned_docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md +++ b/versioned_docs/version-4.x/lakehouse/catalogs/maxcompute-catalog.md @@ -122,7 +122,7 @@ Starting from Doris 4.1.x, MaxCompute Catalog's external metadata cache is confi ### Cache Property Configuration (4.1.x+) {#meta-cache-unified-model} -Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity}`. +MaxCompute cache entries use the configuration key format `meta.cache.maxcompute..{enable,ttl-second,capacity}`. MaxCompute entries are currently count-bounded and do not accept entry-level `max-weight`; see [External Metadata Cache Memory Management](../external-meta-cache-memory-management.md). | Property | Example | Meaning | |---|---|---| @@ -156,7 +156,7 @@ In version 4.1.x and later, unified keys are recommended. The following is the m -- Disable partition value cache to detect the latest partitions in MaxCompute tables ALTER CATALOG mc_ctl SET PROPERTIES ("meta.cache.maxcompute.partition_values.ttl-second" = "0"); ``` -* **Note**: `meta.cache.maxcompute.*` currently does not have a dedicated hot-reload hook. After changing the configuration, it is recommended to recreate the Catalog or restart FE to ensure it takes effect. +* **Note**: Starting from Doris 4.1.4, a successful Catalog property change resets the Catalog execution context and clears initialized entries for every metadata-cache engine routed by that Catalog. Doris rebuilds them with the new context and configuration on the next access; in-flight queries are not affected. ### Observability {#meta-cache-unified-observability} diff --git a/versioned_docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx b/versioned_docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx index fe04ba21cf0af..18b9adc637090 100644 --- a/versioned_docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx +++ b/versioned_docs/version-4.x/lakehouse/catalogs/paimon-catalog.mdx @@ -231,7 +231,7 @@ CREATE CATALOG [IF NOT EXISTS] catalog_name PROPERTIES ( ## Metadata Cache {#meta-cache} -To improve the performance of accessing external data sources, Apache Doris caches Paimon metadata. Metadata includes table structure (Schema) and table objects. +To improve the performance of accessing external data sources, Apache Doris caches Paimon metadata. Metadata includes table structure (Schema), table objects, snapshots, and partition projections. :::tip For versions before Doris 4.1.x, metadata caching is mainly controlled globally by FE configuration items. For details, see [Metadata Cache](../meta-cache.md). @@ -240,15 +240,16 @@ Starting from Doris 4.1.x, Paimon Catalog's external metadata cache is configure ### Cache Property Configuration (4.1.x+) {#meta-cache-unified-model} -Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity}`. +Each engine's cache entry uses a unified configuration key format: `meta.cache...{enable,ttl-second,capacity,max-weight}`. Memory limits apply to the estimator-backed `table` and `snapshot` entries; see [External Metadata Cache Memory Management](../external-meta-cache-memory-management.md). | Property | Example | Meaning | |---|---|---| | `enable` | `true/false` | Whether to enable this cache module. | | `ttl-second` | `600`, `0`, `-1` | `0` means disable cache (takes effect immediately, can be used to see the latest data); `-1` means never expire; other positive integers mean TTL in seconds based on access time. | -| `capacity` | `10000` | Maximum number of cache entries (by count). `0` means disable. | +| `capacity` | `10000` | Maximum number of cache entries by count. When `max-weight` is active, it is not a simultaneous count limit, but `0` still disables the cache. | +| `max-weight` | `1GB` | Optional estimated retained-memory limit. Supported by Paimon `table` and `snapshot`; `0` disables that entry. | -**Effective Logic:** The module cache only takes effect when `enable=true`, `ttl-second != 0`, and `capacity > 0`. +**Effective Logic:** The module cache takes effect when `enable=true`, `ttl-second != 0`, `capacity > 0`, and `max-weight` is either absent or not `0`. ### Cache Modules {#meta-cache-unified-modules} @@ -258,6 +259,7 @@ Paimon Catalog includes the following cache modules: |---|---|---| | `schema` | `meta.cache.paimon.schema.` | Caches table structure. Impact: Visibility of table column information. If disabled, the latest Schema is pulled for each query. | | `table` | `meta.cache.paimon.table.` | Caches Paimon table metadata objects. Impact: Reduces metadata loading overhead during query planning. | +| `snapshot` | `meta.cache.paimon.snapshot.` | Caches snapshot metadata, schema generations, and partition projections. This entry supports `max-weight`. | ### Legacy Parameter Mapping and Conversion {#meta-cache-mapping} @@ -274,7 +276,7 @@ In version 4.1.x and later, unified keys are recommended. The following is the m -- Disable table object cache to detect the latest snapshots of Paimon tables ALTER CATALOG paimon_ctl SET PROPERTIES ("meta.cache.paimon.table.ttl-second" = "0"); ``` -* **Performance optimization**: Changes via `ALTER CATALOG ... SET PROPERTIES` support hot-reload in Paimon Catalog. +* **Performance optimization**: Starting from Doris 4.1.4, a successful Catalog property change resets the Catalog execution context and clears initialized entries for every metadata-cache engine routed by that Catalog. Doris rebuilds them with the new context and configuration on the next access; in-flight queries are not affected. When Doris memory governance is enabled, Paimon's SDK `CachingCatalog` is disabled by default; an explicit `paimon.cache-enabled` setting takes precedence. ### Observability {#meta-cache-unified-observability} @@ -293,7 +295,7 @@ See the documentation for this system table: [catalog_meta_cache_statistics](../ ### Supported Paimon Versions -The currently dependent Paimon version is 1.0.0. +The currently dependent Paimon version for Doris 4.1.4 is 1.4.2. ### Supported Paimon Formats diff --git a/versioned_docs/version-4.x/lakehouse/external-meta-cache-memory-management.md b/versioned_docs/version-4.x/lakehouse/external-meta-cache-memory-management.md new file mode 100644 index 0000000000000..fc7ba469f28b9 --- /dev/null +++ b/versioned_docs/version-4.x/lakehouse/external-meta-cache-memory-management.md @@ -0,0 +1,289 @@ +--- +{ + "title": "External Metadata Cache Memory Management", + "language": "en", + "description": "Describes FE, Catalog, and cache-entry memory limits for external metadata caches in Doris 4.1.4, including Hive, Iceberg, and Paimon configuration." +} +--- + +Starting from Doris 4.1.4, selected external metadata caches can be limited by estimated retained memory. This prevents metadata from a large table, a single Catalog, or multiple Catalogs from continuously consuming FE heap. + +Use this feature when: + +- Hive tables have enough partitions to create large partition-pruning structures; +- Iceberg table metadata, snapshots, or manifests are large; +- Paimon table generations or snapshots contain large retained metadata and partition projections; +- an FE serves multiple Catalogs and one Catalog must not dominate metadata-cache memory. + +:::caution +These limits cover only the estimator-backed cache entries listed in this document. They are not a hard FE heap limit. Shared runtime infrastructure such as Catalog clients and thread pools is not fully attributed to one cache entry, while bounded owner-specific payloads retained by an entry, including authentication context and Iceberg FileIO configuration or credentials, are conservatively charged. Metadata caches that are not estimator-backed remain outside this quota. +::: + +## Quick start + +Prerequisites: + +- an existing Hive, Iceberg, or Paimon Catalog; +- permission to update FE configuration and the target Catalog; +- Doris 4.1.4 or later on every FE. + +### 1. Configure the FE-wide limit + +Set the following in `fe.conf` on every FE: + +```properties +external_meta_cache_max_weight = 10% +``` + +All managed external metadata caches on that FE can then use at most 10% of the JVM maximum heap in total. Restart the FE after changing this setting. + +You can also use a fixed size: + +```properties +external_meta_cache_max_weight = 8GB +``` + +### 2. Limit one Catalog + +The following example limits managed metadata caches in `iceberg_ctl` to 4 GB in total: + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB" +); +``` + +After the change, Doris clears the initialized cache group for this Catalog and recreates it with the new configuration on the next access. + +:::tip +`meta.cache.max-weight` works even when no FE-wide limit is configured. In that case, it limits only this Catalog and does not limit the total across all Catalogs on the FE. +::: + +### 3. Limit individual cache entries + +The following example sets a Catalog limit and individual Iceberg entry limits: + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB", + "meta.cache.iceberg.table.max-weight" = "1GB", + "meta.cache.iceberg.snapshot.max-weight" = "2GB", + "meta.cache.iceberg.manifest.enable" = "true", + "meta.cache.iceberg.manifest.max-weight" = "1GB" +); +``` + +The `table`, `snapshot`, and `manifest` entries are limited to 1 GB, 2 GB, and 1 GB respectively. Their combined usage cannot exceed the 4 GB Catalog limit, and the Catalog is still constrained by the FE-wide limit. + +## Quota hierarchy + +Three levels of memory limits are available: + +| Level | Location | Setting | Scope | +|---|---|---|---| +| FE | `fe.conf` | `external_meta_cache_max_weight` | All managed external metadata caches on the current FE | +| Catalog | Catalog property | `meta.cache.max-weight` | All managed entries in the current Catalog | +| Cache entry | Catalog property | `meta.cache...max-weight` | One cache entry for one engine | + +The effective limit is the minimum of all configured limits. + +| FE limit | Catalog limit | Entry limit | Behavior | +|---|---|---|---| +| Not set | Not set | Not set | Memory accounting is disabled; the existing count and TTL policies remain in use | +| Set | Not set | Not set | All managed entries share the FE quota | +| Not set | Set | Not set | Managed entries in the Catalog share the Catalog quota | +| Not set | Not set | Set | Only that entry is limited; there is no combined FE or Catalog limit | +| Set | Set | Set | All three levels apply and the strictest limit wins | + +The configured hierarchy must satisfy these rules: + +- a Catalog limit cannot exceed the FE-wide limit; +- an entry limit cannot exceed its configured direct parent; +- on heterogeneous FEs, a follower further clamps admission to its local FE limit. + +`CREATE CATALOG` or `ALTER CATALOG` fails when the hierarchy is invalid instead of silently ignoring the property. + +## Value format + +Fixed sizes support these case-insensitive binary units: + +```text +B, KB, MB, GB, TB, PB +``` + +Examples include `512MB` and `4GB`. + +Fixed-size values must use an integer amount. Decimal values are supported only for percentages, such as `12.5%`. + +Only `external_meta_cache_max_weight` accepts a percentage, such as `10%` or `12.5%`. The percentage is calculated from each FE's JVM maximum heap, so FEs with different heap sizes receive different byte limits. + +Zero has different meanings at different levels: + +| Setting | Meaning of `0` | +|---|---| +| `external_meta_cache_max_weight` | Disables the FE-wide quota without disabling caches | +| `meta.cache.max-weight` | Invalid; the Catalog limit must be positive | +| `meta.cache...max-weight` | Disables that cache entry; it does not remove the entry-level override | + +`0%` is invalid. Use the plain value `0` to disable the FE-wide quota. + +## Supported cache entries + +Only the following entries accept `max-weight` in this version: + +| Catalog/engine | Entry | Property | Cached data | +|---|---|---|---| +| Hive | `partition_values` | `meta.cache.hive.partition_values.max-weight` | Partition names, values, pruning indexes, and sorted ranges | +| Iceberg | `table` | `meta.cache.iceberg.table.max-weight` | Iceberg table metadata and the retained metadata generation | +| Iceberg | `snapshot` | `meta.cache.iceberg.snapshot.max-weight` | Snapshot, partition projection, and its retained table generation | +| Iceberg | `manifest` | `meta.cache.iceberg.manifest.max-weight` | Parsed DataFile and DeleteFile lists; this entry is disabled by default and also requires `enable=true` | +| Paimon | `table` | `meta.cache.paimon.table.max-weight` | Retained Paimon table metadata and its generation | +| Paimon | `snapshot` | `meta.cache.paimon.snapshot.max-weight` | Snapshot, schema generation, and partition projection | + +These common entries remain count-bounded and do not accept `max-weight`: + +- Hive `schema`, `partition`, and `file`; +- Iceberg `schema` and `view`; +- Paimon `schema`; +- existing Hudi, MaxCompute, and Doris Catalog cache entries. + +Setting `max-weight` on an unsupported entry causes Catalog creation or alteration to fail. For example, this configuration is invalid: + +```sql +ALTER CATALOG hive_ctl SET PROPERTIES ( + "meta.cache.hive.file.max-weight" = "1GB" +); +``` + +:::note +An HMS Catalog can route Hive, Hudi, and Iceberg metadata caches. If it contains Iceberg tables, the same HMS Catalog can use `meta.cache.iceberg.table.max-weight`, `meta.cache.iceberg.snapshot.max-weight`, and `meta.cache.iceberg.manifest.max-weight`. +::: + +## Examples + +### Use only Catalog limits + +When there is no FE-wide limit, isolate Catalogs independently: + +```sql +ALTER CATALOG hive_prod SET PROPERTIES ( + "meta.cache.max-weight" = "6GB" +); + +ALTER CATALOG iceberg_ad_hoc SET PROPERTIES ( + "meta.cache.max-weight" = "2GB" +); +``` + +The Catalogs are limited to 6 GB and 2 GB respectively, but their combined use on the FE has no common limit. + +### Limit large Hive partition structures + +```sql +ALTER CATALOG hive_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "4GB", + "meta.cache.hive.partition_values.max-weight" = "3GB" +); +``` + +This primarily limits partition-pruning structures created for highly partitioned Hive tables. Hive file-list caches are not included in this quota. + +### Limit Paimon table generations and snapshots + +```sql +ALTER CATALOG paimon_ctl SET PROPERTIES ( + "meta.cache.max-weight" = "2GB", + "meta.cache.paimon.table.max-weight" = "512MB", + "meta.cache.paimon.snapshot.max-weight" = "1536MB" +); +``` + +Paimon supports independent memory limits for both `table` and `snapshot` entries. This example reserves up to 512 MB for retained table generations and 1536 MB for snapshot projections, within the 2 GB Catalog limit. + +### Compatibility with existing cache properties + +For compatibility with existing Catalogs, Iceberg and Paimon `table.enable`, `table.ttl-second`, and `table.capacity` values are also used as defaults for the corresponding snapshot entry when its property is not explicitly set. For example: + +```sql +ALTER CATALOG paimon_ctl SET PROPERTIES ( + "meta.cache.paimon.table.ttl-second" = "600", + "meta.cache.paimon.snapshot.max-weight" = "1536MB" +); +``` + +Here, `table.ttl-second` controls the TTL of both the Paimon table object and snapshot caches, while `snapshot.max-weight` limits only the snapshot cache. + +`max-weight` is not compatibility-mapped. Use the actual estimator-backed entry name: + +- `meta.cache.iceberg.table.max-weight` for Iceberg table objects; +- `meta.cache.iceberg.snapshot.max-weight` for Iceberg snapshots; +- `meta.cache.paimon.table.max-weight` for Paimon table generations; +- `meta.cache.paimon.snapshot.max-weight` for Paimon snapshots. + +### Combine memory and TTL policies + +Memory limits can be used with `enable`, `ttl-second`, and `capacity`: + +```sql +ALTER CATALOG iceberg_ctl SET PROPERTIES ( + "meta.cache.iceberg.snapshot.enable" = "true", + "meta.cache.iceberg.snapshot.ttl-second" = "1800", + "meta.cache.iceberg.snapshot.capacity" = "1000", + "meta.cache.iceberg.snapshot.max-weight" = "2GB" +); +``` + +When `max-weight` is active, eviction is bounded by memory weight and `capacity` is no longer a simultaneous maximum entry count. However, `capacity=0` still disables the cache, so keep it greater than zero when using a memory limit. + +## Behavior when a limit is reached + +On a cache miss, Doris: + +1. loads metadata from the external source; +2. freezes or materializes the candidate and estimates its retained memory; +3. atomically checks the FE, Catalog, and entry quotas; +4. evicts cold data from the same entry and retries when the quota is insufficient; if local eviction cannot satisfy the reservation, Doris schedules peer reclamation from sibling entries (and, for an FE-wide deficit, other Catalogs); +5. returns the loaded object to the current request without caching it if admission still fails. + +Therefore, a new object that is larger than the available quota does not normally fail the query because of the cache quota. It is not retained, so later accesses may load it again and increase query-planning latency. + +If Doris cannot produce a reliable complete estimate, for example because the retained table form is unsupported, Iceberg metadata is still lazily loaded, preparation fails, or an accounting work budget is exceeded, it also skips caching instead of under-counting the object. The estimators do not perform generic reflective object-graph traversal or container sampling. Iceberg uses one fail-closed private-field probe only to verify that snapshot metadata is already loaded, avoiding estimation-time IO. + +:::note +Local eviction is synchronous. Peer reclamation is asynchronous and best effort, so the current miss can still be returned without caching while sibling entries are reclaimed for later admissions. Configure entry-level limits when one entry must not dominate a shared FE or Catalog quota. +::: + +If a scheduled refresh cannot be admitted because its estimate is incomplete or the budget is insufficient, Doris retains the previous known-good cached generation. The rejected refresh is not published. + +:::caution +Quota checks happen during cache admission, after the object has been loaded and prepared. Doris does not reserve heap before accessing the external source. A remote-load failure, or one exceptionally large object exhausting FE heap before construction completes, can therefore still fail the current request. This feature limits memory retained after a successful load; it is not OOM protection for a single metadata load. +::: + +## Applying configuration and refreshing caches + +- Restart an FE after changing `external_meta_cache_max_weight`; +- changing Catalog properties, including `meta.cache.max-weight` or `meta.cache...*`, resets the Catalog execution context and clears initialized entries for all metadata-cache engines routed by that Catalog; the next access rebuilds them with the new context and configuration; +- queries already in progress can continue with loaded objects while later accesses use the new configuration. + +Memory limits control whether objects are retained. They do not change external metadata and do not replace `REFRESH CATALOG`, TTL, or metadata event synchronization. + +## Caveats + +- `max-weight` is an estimated retained-cache admission budget. Type-specific formulas use rounded-up structural constants calibrated offline plus loader-time payload counters; they do not model the exact layout of the active JVM or third-party SDK. The result is not operating-system RSS and does not replace FE heap and GC monitoring; +- shared runtime infrastructure is not fully charged to an individual entry, but bounded owner-specific authentication context, FileIO configuration, storage credentials, and transform payloads retained by an entry are conservatively included; +- weight-bounded entries use soft values. Reservation records retain the key, generation, and weight, but no strong reference to the cached value. Under heap pressure, the JVM may collect a value before its TTL expires or its quota is full; Doris releases the matching reservation and reloads the value on a later access; +- large objects are estimated only on load, refresh, or replacement. The completed weight is retained with the published generation, so cache hits do not rescan object contents; +- first-time Iceberg table or snapshot admission may materialize the current snapshot's manifest lists and can increase initial load latency; +- newly supplied DDL properties strictly reject misspelled engine names, entry names, and options; legacy persisted properties that cannot be parsed at runtime are warned about and ignored so an upgraded Catalog can still initialize; +- when Doris memory governance is active for an Iceberg Catalog, the Iceberg SDK manifest-content cache is not auto-enabled because it is outside the Doris quota. An explicit `io.manifest.cache-enabled` setting takes precedence; +- when Doris memory governance is active for a Paimon Catalog, Paimon's SDK-level `CachingCatalog` is disabled by default to avoid an unbounded second metadata cache. An explicit `paimon.cache-enabled` setting takes precedence; +- after upgrading Iceberg or Paimon, use the SDK version shipped with the Doris release rather than replacing FE SDK JARs independently. + +## Best practices + +1. Start with an FE-wide limit so the total across Catalogs cannot grow without a bound; +2. add Catalog limits for large production Catalogs that share an FE; +3. add entry-level limits only for entries that dominate usage; every entry does not need its own setting; +4. reserve enough heap for query planning and other FE caches instead of assigning most of the JVM heap to external metadata caches; +5. increase the quota or reduce competing entry limits if the same object is repeatedly reloaded after admission rejection; +6. use a percentage for heterogeneous FE heap sizes, or a fixed size when every FE must use the same byte limit. diff --git a/versioned_docs/version-4.x/lakehouse/meta-cache.md b/versioned_docs/version-4.x/lakehouse/meta-cache.md index 2857dcd989c77..f9358a9dda555 100644 --- a/versioned_docs/version-4.x/lakehouse/meta-cache.md +++ b/versioned_docs/version-4.x/lakehouse/meta-cache.md @@ -19,6 +19,8 @@ This article details the types, strategies, and related parameter configurations For **data cache**, refer to the [data cache documentation](./data-cache.md). +For Doris 4.1.4 external metadata cache memory limits, refer to [External Metadata Cache Memory Management](./external-meta-cache-memory-management.md). + ## Cache Strategies Most caches have the following three strategy indicators: diff --git a/versioned_sidebars/version-4.x-sidebars.json b/versioned_sidebars/version-4.x-sidebars.json index b3eda778b5317..b0ae51571eb95 100644 --- a/versioned_sidebars/version-4.x-sidebars.json +++ b/versioned_sidebars/version-4.x-sidebars.json @@ -871,6 +871,7 @@ }, "lakehouse/data-cache", "lakehouse/meta-cache", + "lakehouse/external-meta-cache-memory-management", "lakehouse/compute-node", "lakehouse/statistics", {