From 9cfa41ec34cc8c3f62b8c01c97c8745df66849fa Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Wed, 8 Jul 2026 10:23:58 +0800 Subject: [PATCH 01/11] [refactor](be) Add file meta cache persistent interface ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: External file metadata disk cache should be built behind the FileMetaCache boundary instead of exposing a generic complete-value put/get API from BlockFileCache. This change adds a narrow FileMetaCache persistent cache interface, lookup/insert result types, profile counter plumbing, and config gates so upper-layer metadata cache code can depend on FileMetaCache APIs while the storage implementation remains replaceable. ### Release note None ### Check List (For Author) - Test: Unit Test - Added focused FileMetaCache interface tests. Attempted targeted BE UT; initial run hit stale PCH after branch switch, clean run was stopped because it triggered a large full rebuild before reaching the target tests. - Behavior changed: No - Does this need documentation: No --- be/src/common/config.cpp | 2 + be/src/common/config.h | 2 + be/src/io/fs/file_meta_cache.cpp | 110 ++++++++++++++++++ be/src/io/fs/file_meta_cache.h | 110 +++++++++++++++++- .../file_reader/file_meta_cache_test.cpp | 73 +++++++++++- 5 files changed, 295 insertions(+), 2 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9ec587bd100ec5..94fad9871a20f1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1213,6 +1213,8 @@ DEFINE_Bool(enable_file_cache, "false"); // Both will use the directory "memory" on the disk instead of the real RAM. DEFINE_String(file_cache_path, "[{\"path\":\"${DORIS_HOME}/file_cache\"}]"); DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB +DEFINE_Bool(enable_external_file_meta_disk_cache, "false"); +DEFINE_mInt64(external_file_meta_disk_cache_max_entry_bytes, "67108864"); // 64MB DEFINE_Bool(clear_file_cache, "false"); DEFINE_mBool(enable_file_cache_query_limit, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 012f09a735c0cd..d8e4a4cf48fbcb 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1258,6 +1258,8 @@ DECLARE_Bool(enable_file_cache); // Both will use the directory "memory" on the disk instead of the real RAM. DECLARE_String(file_cache_path); DECLARE_Int64(file_cache_each_block_size); +DECLARE_Bool(enable_external_file_meta_disk_cache); +DECLARE_mInt64(external_file_meta_disk_cache_max_entry_bytes); DECLARE_Bool(clear_file_cache); DECLARE_mBool(enable_file_cache_query_limit); DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent); diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index f97b2f80cd6ee6..db5e04f63ec44f 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -17,7 +17,26 @@ #include "io/fs/file_meta_cache.h" +#include + +#include "common/config.h" +#include "common/logging.h" +#include "util/stopwatch.hpp" + namespace doris { +namespace { + +void update_profile_counter(int64_t* counter, int64_t value = 1) { + if (counter != nullptr) { + *counter += value; + } +} + +} // namespace + +FileMetaCache::FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache) + : _cache(capacity), _persistent_cache(std::move(persistent_cache)) {} std::string FileMetaCache::get_key(const std::string file_name, int64_t modification_time, int64_t file_size) { @@ -40,4 +59,95 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } +bool FileMetaCache::is_persistent_cache_enabled() { + return config::enable_external_file_meta_disk_cache && + config::external_file_meta_disk_cache_max_entry_bytes > 0; +} + +bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) { + const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; + return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 && + payload_size <= static_cast(max_entry_bytes); +} + +FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, + std::string* serialized_meta, + FileMetaCacheProfile* profile) { + DCHECK(handle != nullptr); + DCHECK(serialized_meta != nullptr); + if (context.enable_memory_cache && lookup(context.key, handle)) { + serialized_meta->clear(); + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_memory_cache); + } + return {.state = FileMetaCacheLookupState::MEMORY_HIT}; + } + + FileMetaCacheLookupResult result; + int64_t persisted_read_time = 0; + if (lookup_persistent_cache(context, serialized_meta, &persisted_read_time)) { + result.state = FileMetaCacheLookupState::PERSISTED_HIT; + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); + } + } else if (is_persistent_cache_enabled() && profile != nullptr) { + update_profile_counter(profile->miss_disk_cache); + } + return result; +} + +void FileMetaCache::invalidate_persistent_cache(const FileMetaCacheContext& context) { + if (_persistent_cache != nullptr) { + _persistent_cache->remove(context.format, context.key); + } +} + +bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context, + std::string* payload, int64_t* read_time) { + DCHECK(payload != nullptr); + DCHECK(read_time != nullptr); + payload->clear(); + *read_time = 0; + if (!is_persistent_cache_enabled() || _persistent_cache == nullptr) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->read(context.format, context.key, context.modification_time, + context.file_size, payload); + *read_time = watch.elapsed_time(); + if (!status.ok()) { + payload->clear(); + VLOG_DEBUG << "lookup file meta persistent cache failed: " << status; + return false; + } + return true; +} + +bool FileMetaCache::insert_persistent_cache(const FileMetaCacheContext& context, + std::string_view payload, int64_t* write_time) { + DCHECK(write_time != nullptr); + *write_time = 0; + if (!is_persistent_cache_payload_size_allowed(static_cast(payload.size())) || + _persistent_cache == nullptr) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->write(context.format, context.key, context.modification_time, + context.file_size, payload); + *write_time = watch.elapsed_time(); + if (!status.ok()) { + VLOG_DEBUG << "insert file meta persistent cache failed: " << status; + return false; + } + return true; +} + } // namespace doris diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 0c62c963ce122d..a7b18f94b924c6 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -17,18 +17,77 @@ #pragma once +#include +#include +#include +#include + +#include "common/status.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" #include "util/obj_lru_cache.h" namespace doris { +enum class FileMetaCacheFormat : uint8_t { + PARQUET = 1, + ORC = 2, +}; + +struct FileMetaCacheContext { + FileMetaCacheFormat format; + const std::string& key; + int64_t modification_time = 0; + int64_t file_size = 0; + bool enable_memory_cache = true; +}; + +enum class FileMetaCacheLookupState { + MEMORY_HIT, + PERSISTED_HIT, + MISS, +}; + +struct FileMetaCacheLookupResult { + FileMetaCacheLookupState state = FileMetaCacheLookupState::MISS; +}; + +struct FileMetaCacheInsertResult { + bool memory_inserted = false; + bool persisted_inserted = false; +}; + +struct FileMetaCacheProfile { + int64_t* hit_cache = nullptr; + int64_t* hit_memory_cache = nullptr; + int64_t* hit_disk_cache = nullptr; + int64_t* miss_disk_cache = nullptr; + int64_t* write_disk_cache = nullptr; + int64_t* read_disk_cache_time = nullptr; + int64_t* write_disk_cache_time = nullptr; +}; + +class FileMetaPersistentCache { +public: + virtual ~FileMetaPersistentCache() = default; + + virtual Status read(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, std::string* payload) = 0; + + virtual Status write(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, + std::string_view payload) = 0; + + virtual void remove(FileMetaCacheFormat format, const std::string& key) = 0; +}; + // A file meta cache depends on a LRU cache. // Such as parsed parquet footer. // The capacity will limit the number of cache entries in cache. class FileMetaCache { public: - FileMetaCache(int64_t capacity) : _cache(capacity) {} + explicit FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache = nullptr); FileMetaCache(const FileMetaCache&) = delete; const FileMetaCache& operator=(const FileMetaCache&) = delete; @@ -41,6 +100,9 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); + static bool is_persistent_cache_enabled(); + static bool is_persistent_cache_payload_size_allowed(uint64_t payload_size); + bool lookup(const std::string& key, ObjLRUCache::CacheHandle* handle) { return _cache.lookup({key}, handle); } @@ -50,10 +112,56 @@ class FileMetaCache { _cache.insert({key}, value, handle); } + template + bool insert(const std::string& key, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle) { + DCHECK(value != nullptr); + if (!_cache.enabled()) { + return false; + } + _cache.insert({key}, value.release(), handle); + return true; + } + bool enabled() const { return _cache.enabled(); } + FileMetaCacheLookupResult lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, std::string* serialized_meta, + FileMetaCacheProfile* profile = nullptr); + + void invalidate_persistent_cache(const FileMetaCacheContext& context); + + template + FileMetaCacheInsertResult insert(const FileMetaCacheContext& context, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle, + std::string_view serialized_meta, + FileMetaCacheProfile* profile = nullptr) { + FileMetaCacheInsertResult result; + int64_t persisted_write_time = 0; + result.persisted_inserted = + insert_persistent_cache(context, serialized_meta, &persisted_write_time); + if (result.persisted_inserted && profile != nullptr) { + if (profile->write_disk_cache != nullptr) { + ++(*profile->write_disk_cache); + } + if (profile->write_disk_cache_time != nullptr) { + *profile->write_disk_cache_time += persisted_write_time; + } + } + if (context.enable_memory_cache) { + result.memory_inserted = insert(context.key, value, handle); + } + return result; + } + private: + bool lookup_persistent_cache(const FileMetaCacheContext& context, std::string* payload, + int64_t* read_time); + bool insert_persistent_cache(const FileMetaCacheContext& context, std::string_view payload, + int64_t* write_time); + ObjLRUCache _cache; + std::unique_ptr _persistent_cache; }; } // namespace doris diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 4f8f803bd3ef28..07eb171e9ebc5d 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -17,9 +17,13 @@ #include "io/fs/file_meta_cache.h" +#include + +#include "common/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "io/fs/file_reader.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -155,4 +159,71 @@ TEST(FileMetaCacheTest, InsertAndLookupWithIntValue) { EXPECT_EQ(*cached_val2, 12345); } -} // namespace doris \ No newline at end of file +TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE(cache.insert(meta_key, value, &insert_handle)); + ASSERT_TRUE(insert_handle.valid()); + + int64_t hit_cache = 0; + int64_t hit_memory_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_memory_cache = &hit_memory_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache}; + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + const FileMetaCacheLookupResult result = + cache.lookup(context, &lookup_handle, &serialized_meta, &profile); + + EXPECT_EQ(result.state, FileMetaCacheLookupState::MEMORY_HIT); + EXPECT_TRUE(lookup_handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_memory_cache, 1); + EXPECT_EQ(hit_disk_cache, 0); + EXPECT_EQ(miss_disk_cache, 0); +} + +TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = false; + + FileMetaCache cache(1024 * 1024); + const std::string meta_key = + FileMetaCache::get_key("s3://bucket/skip-memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult result = + cache.insert(context, value, &handle, std::string_view(*value)); + + EXPECT_FALSE(result.persisted_inserted); + EXPECT_FALSE(result.memory_inserted); + EXPECT_FALSE(handle.valid()); + EXPECT_NE(value, nullptr); + + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); +} + +} // namespace doris From 283f9408322b417a1af9d3a74c1617096a578f01 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Wed, 8 Jul 2026 10:54:26 +0800 Subject: [PATCH 02/11] [improvement](be) Add disk cache for external file metadata ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: External Parquet and ORC readers only kept file metadata in the in-memory FileMetaCache. This change adds a FileMetaDiskCache implementation behind the FileMetaCache persistent-cache interface, stores serialized Parquet footers and ORC file tails in the existing BlockFileCache through its block APIs, and wires readers to read from disk cache, refill memory cache, and write new persistent entries on misses. It also wires the v2 Parquet reader path through FileScannerV2/TableReader so current Parquet scans use the file meta cache, and enables external file meta disk cache by default. ### Release note Enable disk cache for external Parquet/ORC file metadata by default. ### Check List (For Author) - Test: Regression test / Unit Test / Manual test - python3 build-support/run_clang_format.py --clang-format-executable clang-format --style file --inplace true -j 1 - git diff --check -- - ./run-be-ut.sh --run --filter=NewParquetReaderTest.UsesFileMetaCacheForFooterMetadata -j 4 - ./run-be-ut.sh --run --filter=FileMetaCache* -j 4 - ./build.sh --be -j 4 - Manual e2e with output FE/BE: local Parquet TVF returned 30000 before and after BE restart, local ORC TVF returned 10, BE runtime config showed enable_external_file_meta_disk_cache=true, and the post-restart Parquet query added no new file meta disk cache miss log. - Behavior changed: Yes. External Parquet/ORC metadata disk cache is enabled by default and can persist file metadata through the file cache. - Does this need documentation: No --- be/src/common/config.cpp | 2 +- be/src/exec/scan/file_scanner.h | 3 +- be/src/exec/scan/file_scanner_v2.cpp | 9 +- be/src/format/orc/vorc_reader.cpp | 68 +++- be/src/format/orc/vorc_reader.h | 12 + be/src/format/parquet/vparquet_reader.cpp | 98 +++++- be/src/format/parquet/vparquet_reader.h | 12 + .../parquet/parquet_file_context.cpp | 78 ++++- .../format_v2/parquet/parquet_file_context.h | 5 +- be/src/format_v2/parquet/parquet_reader.cpp | 18 +- be/src/format_v2/parquet/parquet_reader.h | 5 +- be/src/format_v2/table_reader.cpp | 3 +- be/src/format_v2/table_reader.h | 3 + be/src/io/fs/file_meta_disk_cache.cpp | 297 ++++++++++++++++++ be/src/io/fs/file_meta_disk_cache.h | 52 +++ be/src/runtime/exec_env_init.cpp | 10 +- .../file_reader/file_meta_cache_test.cpp | 55 ++++ .../format_v2/parquet/parquet_reader_test.cpp | 28 +- 18 files changed, 722 insertions(+), 36 deletions(-) create mode 100644 be/src/io/fs/file_meta_disk_cache.cpp create mode 100644 be/src/io/fs/file_meta_disk_cache.h diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 94fad9871a20f1..3b7745e3b97c0a 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1213,7 +1213,7 @@ DEFINE_Bool(enable_file_cache, "false"); // Both will use the directory "memory" on the disk instead of the real RAM. DEFINE_String(file_cache_path, "[{\"path\":\"${DORIS_HOME}/file_cache\"}]"); DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB -DEFINE_Bool(enable_external_file_meta_disk_cache, "false"); +DEFINE_Bool(enable_external_file_meta_disk_cache, "true"); DEFINE_mInt64(external_file_meta_disk_cache_max_entry_bytes, "67108864"); // 64MB DEFINE_Bool(clear_file_cache, "false"); diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 3675fd2449711e..2781771685f190 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -331,7 +331,8 @@ class FileScanner : public Scanner { // 2. the file number is less than 1/3 of cache's capacibility // Otherwise, the cache miss rate will be high bool _should_enable_file_meta_cache() { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + return (ExecEnv::GetInstance()->file_meta_cache()->enabled() || + FileMetaCache::is_persistent_cache_enabled()) && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 10b5f850ea36f7..34b4eb758ec516 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -438,6 +438,11 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + FileMetaCache* file_meta_cache = nullptr; + if (_should_enable_file_meta_cache()) { + file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + DORIS_CHECK(file_meta_cache != nullptr); + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), @@ -449,6 +454,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), .condition_cache_digest = _local_state->get_condition_cache_digest(), + .file_meta_cache = file_meta_cache, })); return Status::OK(); } @@ -512,7 +518,8 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not } bool FileScannerV2::_should_enable_file_meta_cache() const { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + return (ExecEnv::GetInstance()->file_meta_cache()->enabled() || + FileMetaCache::is_persistent_cache_enabled()) && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..c4907bc3961072 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -336,6 +336,18 @@ void OrcReader::_collect_profile_before_close() { COUNTER_UPDATE(_orc_profile.lazy_read_filtered_rows, _statistics.lazy_read_filtered_rows); COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _statistics.file_footer_read_calls); COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _statistics.file_footer_write_disk_cache_time); if (_file_input_stream != nullptr) { _file_input_stream->collect_profile_before_close(); } @@ -376,6 +388,18 @@ void OrcReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _orc_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); } } @@ -439,21 +463,53 @@ Status OrcReader::_create_file_reader() { RETURN_IF_ERROR(create_orc_reader()); } else { auto inner_file_reader = _file_input_stream->get_inner_reader(); - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(inner_file_reader, _file_description); + const int64_t file_size = _file_description.file_size == -1 ? inner_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_statistics.file_footer_hit_cache, + .hit_memory_cache = &_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_statistics.file_footer_write_disk_cache_time}; // Local variables can be required because setSerializedFileTail is an assignment operation, not a reference. ObjLRUCache::CacheHandle _meta_cache_handle; - if (_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { - const std::string* footer_ptr = _meta_cache_handle.data(); + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const std::string* footer_ptr = _meta_cache_handle.data(); options.setSerializedFileTail(*footer_ptr); RETURN_IF_ERROR(create_orc_reader()); - _statistics.file_footer_hit_cache++; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto footer_ptr = std::make_unique(std::move(serialized_file_tail)); + options.setSerializedFileTail(*footer_ptr); + RETURN_IF_ERROR(create_orc_reader()); + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(file_meta_cache_key, footer_ptr, &_meta_cache_handle); + } } else { _statistics.file_footer_read_calls++; RETURN_IF_ERROR(create_orc_reader()); - std::string* footer_ptr = new std::string {_reader->getSerializedFileTail()}; - _meta_cache->insert(file_meta_cache_key, footer_ptr, &_meta_cache_handle); + const uint64_t file_tail_size = + _reader->getFileFooterLength() + _reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto footer_ptr = std::make_unique(_reader->getSerializedFileTail()); + _meta_cache->insert(file_meta_cache_context, footer_ptr, &_meta_cache_handle, + *footer_ptr, &file_meta_cache_profile); + } } } diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 6d64f604cdee5a..e8c006e33dc2fe 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -150,6 +150,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { int64_t lazy_read_filtered_rows = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; }; OrcReader(RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, @@ -317,6 +323,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { RuntimeProfile::Counter* evaluated_row_group_count = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class ORCFilterImpl : public orc::ORCFilter { diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 26caf7faac66f2..8dc3dd7154ebcc 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -65,6 +65,7 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "util/slice.h" #include "util/string_util.h" +#include "util/thrift_util.h" #include "util/timezone_utils.h" namespace cctz { @@ -255,6 +256,18 @@ void ParquetReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _parquet_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_read_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + _parquet_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); _parquet_profile.decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime", parquet_profile, 1); _parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -367,21 +380,76 @@ Status ParquetReader::_open_file() { // parse magic number & parse meta data _reader_statistics.file_footer_read_calls += 1; } else { - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(_tracing_file_reader, _file_description); - if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { + const int64_t file_size = _file_description.file_size == -1 + ? _tracing_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + std::string footer_payload; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &footer_payload, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + _file_metadata = _meta_cache_handle.data(); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + uint32_t metadata_size = static_cast(footer_payload.size()); + tparquet::FileMetaData t_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg( + reinterpret_cast(footer_payload.data()), &metadata_size, + true, &t_metadata)); + _file_metadata_ptr = std::make_unique(t_metadata, metadata_size); + RETURN_IF_ERROR(_file_metadata_ptr->init_schema(enable_mapping_varbinary, + enable_mapping_timestamp_tz)); + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), + &_meta_cache_handle); + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, enable_mapping_timestamp_tz)); - // _file_metadata_ptr.release() : move control of _file_metadata to _meta_cache_handle - _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), - &_meta_cache_handle); - _file_metadata = _meta_cache_handle.data(); + const size_t serialized_meta_size = meta_size >= PARQUET_FOOTER_SIZE + ? meta_size - PARQUET_FOOTER_SIZE + : meta_size; + if (FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(serialized_meta_size))) { + tparquet::FileMetaData thrift_metadata = _file_metadata_ptr->to_thrift(); + ThriftSerializer serializer(true, static_cast(serialized_meta_size)); + RETURN_IF_ERROR(serializer.serialize(&thrift_metadata, &footer_payload)); + const auto insert_result = _meta_cache->insert( + file_meta_cache_context, _file_metadata_ptr, &_meta_cache_handle, + footer_payload, &file_meta_cache_profile); + if (insert_result.memory_inserted) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else if (file_meta_cache_context.enable_memory_cache && + _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr, + &_meta_cache_handle)) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } _reader_statistics.file_footer_read_calls += 1; - } else { - _reader_statistics.file_footer_hit_cache++; } - _file_metadata = _meta_cache_handle.data(); } if (_file_metadata == nullptr) { @@ -1742,6 +1810,18 @@ void ParquetReader::_collect_profile() { _reader_statistics.file_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); COUNTER_UPDATE(_parquet_profile.skip_page_header_num, _column_statistics.skip_page_header_num); COUNTER_UPDATE(_parquet_profile.parse_page_header_num, diff --git a/be/src/format/parquet/vparquet_reader.h b/be/src/format/parquet/vparquet_reader.h index 9046641c6c0c51..e6c768c2847784 100644 --- a/be/src/format/parquet/vparquet_reader.h +++ b/be/src/format/parquet/vparquet_reader.h @@ -108,6 +108,12 @@ class ParquetReader : public TableFormatReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; @@ -281,6 +287,12 @@ class ParquetReader : public TableFormatReader { RuntimeProfile::Counter* parse_page_index_time = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* decompress_time = nullptr; RuntimeProfile::Counter* decompress_cnt = nullptr; RuntimeProfile::Counter* page_read_counter = nullptr; diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index d80dc58181d4f6..81af94878ca155 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -35,6 +35,7 @@ #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" @@ -533,19 +534,80 @@ Status arrow_status_to_doris_status(const arrow::Status& status) { } Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { + bool enable_page_cache, const io::FileDescription& file_description, + FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, + int64_t* file_footer_read_calls) { DORIS_CHECK(input_file_reader != nullptr); + DORIS_CHECK(file_footer_read_calls != nullptr); auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); + const std::string file_meta_cache_key = + file_meta_cache != nullptr ? FileMetaCache::get_key(input_file_reader, file_description) + : std::string {}; + const int64_t file_size = file_description.file_size == -1 + ? static_cast(input_file_reader->size()) + : file_description.file_size; + std::shared_ptr<::parquet::FileMetaData> cached_metadata; + bool metadata_from_cache = false; + FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET, + .key = file_meta_cache_key, + .modification_time = file_description.mtime, + .file_size = file_size, + .enable_memory_cache = file_meta_cache != nullptr && file_meta_cache->enabled()}; + const auto reader_properties = ::parquet::default_reader_properties(); try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); + if (file_meta_cache != nullptr) { + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_metadata; + const FileMetaCacheLookupResult lookup_result = + file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_metadata, file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_metadata_ptr = + meta_cache_handle.data>(); + DORIS_CHECK(cached_metadata_ptr != nullptr); + cached_metadata = *cached_metadata_ptr; + metadata_from_cache = true; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + uint32_t metadata_len = static_cast(serialized_metadata.size()); + cached_metadata = ::parquet::FileMetaData::Make(serialized_metadata.data(), + &metadata_len, reader_properties); + if (file_meta_cache_context.enable_memory_cache) { + auto cached_metadata_holder = + std::make_unique>( + cached_metadata); + ObjLRUCache::CacheHandle insert_handle; + file_meta_cache->insert(file_meta_cache_key, cached_metadata_holder, + &insert_handle); + } + metadata_from_cache = true; + } + } + arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, + enable_page_cache, + std::move(page_cache_file_key)); + this->file_reader = + ::parquet::ParquetFileReader::Open(arrow_file, reader_properties, cached_metadata); metadata = this->file_reader->metadata(); schema = metadata != nullptr ? metadata->schema() : nullptr; + if (!metadata_from_cache) { + ++(*file_footer_read_calls); + if (file_meta_cache != nullptr && metadata != nullptr) { + auto cached_metadata_holder = + std::make_unique>(metadata); + ObjLRUCache::CacheHandle insert_handle; + if (FileMetaCache::is_persistent_cache_payload_size_allowed(metadata->size())) { + std::string serialized_metadata = metadata->SerializeToString(); + file_meta_cache->insert(file_meta_cache_context, cached_metadata_holder, + &insert_handle, serialized_metadata, + file_meta_cache_profile); + } else if (file_meta_cache_context.enable_memory_cache) { + file_meta_cache->insert(file_meta_cache_key, cached_metadata_holder, + &insert_handle); + } + } + } } catch (const ::parquet::ParquetException& e) { if (io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()).find("stop") != std::string_view::npos) { diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..59ae5a9d0ce057 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -32,6 +32,8 @@ struct IOContext; } // namespace doris::io namespace doris { +class FileMetaCache; +struct FileMetaCacheProfile; class RuntimeProfile; } // namespace doris @@ -112,7 +114,8 @@ struct ParquetFileContext { const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); + const io::FileDescription& file_description, FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, int64_t* file_footer_read_calls); // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page // decoding, so v2 caches the serialized bytes read inside these ranges and excludes // footer/metadata reads that happen before registration. diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 82ad3f78891c80..bc70d943f470d7 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -40,6 +40,7 @@ #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/column_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -312,10 +313,11 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache) {} ParquetReader::~ParquetReader() = default; @@ -348,8 +350,18 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); // Open parquet file and parse metadata to get file schema. + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache}; RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + _state->enable_page_cache, *_file_description, + _file_meta_cache, &file_meta_cache_profile, + &_reader_statistics.file_footer_read_calls)); + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _reader_statistics.file_footer_hit_cache); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier RETURN_IF_ERROR( diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..8f88e2f5fb92b6 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -27,6 +27,7 @@ #include "format_v2/parquet/parquet_scan.h" namespace doris { +class FileMetaCache; namespace io { struct IOContext; } // namespace io @@ -46,7 +47,8 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, + FileMetaCache* file_meta_cache = nullptr); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +91,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + FileMetaCache* _file_meta_cache = nullptr; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 11e2b30df6de23..05b3d4d8117175 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -470,6 +470,7 @@ Status TableReader::init(TableReadOptions&& options) { _io_ctx = options.io_ctx; _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; + _file_meta_cache = options.file_meta_cache; _file_slot_descs = options.file_slot_descs; _push_down_agg_type = options.push_down_agg_type; _condition_cache_digest = options.condition_cache_digest; @@ -677,7 +678,7 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { if (_format == FileFormat::PARQUET) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache); return Status::OK(); } if (_format == FileFormat::ORC) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index da2cb351ff68ce..37ff0011530568 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -69,6 +69,7 @@ namespace doris { class Block; +class FileMetaCache; struct DeleteFileDesc; class RuntimeState; } // namespace doris @@ -137,6 +138,7 @@ struct TableReadOptions { const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; // Digest of stable pushed-down predicates. A zero digest disables condition cache. uint64_t condition_cache_digest = 0; + FileMetaCache* file_meta_cache = nullptr; }; struct SplitReadOptions { @@ -1525,6 +1527,7 @@ class TableReader { std::shared_ptr _io_ctx; RuntimeState* _runtime_state; RuntimeProfile* _scanner_profile; + FileMetaCache* _file_meta_cache = nullptr; const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp new file mode 100644 index 00000000000000..d20316917e3700 --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/file_meta_disk_cache.h" + +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +namespace { + +constexpr std::string_view FILE_META_DISK_CACHE_MAGIC = "DFMC"; +constexpr uint8_t FILE_META_DISK_CACHE_VERSION = 1; +constexpr size_t FILE_META_DISK_CACHE_HEADER_SIZE = 4 + 1 + 1 + 2 + 8 + 8 + 8 + 4; + +std::string_view format_name(FileMetaCacheFormat format) { + switch (format) { + case FileMetaCacheFormat::PARQUET: + return "parquet"; + case FileMetaCacheFormat::ORC: + return "orc"; + } + DCHECK(false) << "unknown file meta cache format"; + return "unknown"; +} + +Status parse_format(uint8_t format, FileMetaCacheFormat* parsed) { + if (format == static_cast(FileMetaCacheFormat::PARQUET)) { + *parsed = FileMetaCacheFormat::PARQUET; + return Status::OK(); + } + if (format == static_cast(FileMetaCacheFormat::ORC)) { + *parsed = FileMetaCacheFormat::ORC; + return Status::OK(); + } + return Status::NotFound("file meta disk cache format mismatch"); +} + +struct FileMetaDiskCacheHeader { + FileMetaCacheFormat format; + int64_t modification_time = 0; + int64_t file_size = 0; + uint64_t payload_size = 0; + uint32_t checksum = 0; +}; + +Status parse_header(std::string_view header, FileMetaDiskCacheHeader* parsed) { + DCHECK(header.size() == FILE_META_DISK_CACHE_HEADER_SIZE); + if (std::memcmp(header.data(), FILE_META_DISK_CACHE_MAGIC.data(), + FILE_META_DISK_CACHE_MAGIC.size()) != 0) { + return Status::NotFound("file meta disk cache magic mismatch"); + } + + const auto* ptr = + reinterpret_cast(header.data() + FILE_META_DISK_CACHE_MAGIC.size()); + const uint8_t version = *ptr++; + if (version != FILE_META_DISK_CACHE_VERSION) { + return Status::NotFound("file meta disk cache version mismatch"); + } + + RETURN_IF_ERROR(parse_format(*ptr++, &parsed->format)); + ptr += 2; + parsed->file_size = static_cast(decode_fixed64_le(ptr)); + ptr += sizeof(uint64_t); + parsed->modification_time = static_cast(decode_fixed64_le(ptr)); + ptr += sizeof(uint64_t); + parsed->payload_size = decode_fixed64_le(ptr); + ptr += sizeof(uint64_t); + parsed->checksum = decode_fixed32_le(ptr); + return Status::OK(); +} + +std::string build_cache_value(FileMetaCacheFormat format, int64_t modification_time, + int64_t file_size, std::string_view payload) { + std::string value; + value.reserve(FILE_META_DISK_CACHE_HEADER_SIZE + payload.size()); + value.append(FILE_META_DISK_CACHE_MAGIC.data(), FILE_META_DISK_CACHE_MAGIC.size()); + value.push_back(static_cast(FILE_META_DISK_CACHE_VERSION)); + value.push_back(static_cast(format)); + value.push_back(0); + value.push_back(0); + put_fixed64_le(&value, static_cast(file_size)); + put_fixed64_le(&value, static_cast(modification_time)); + put_fixed64_le(&value, payload.size()); + put_fixed32_le(&value, crc32c::Crc32c(payload.data(), payload.size())); + value.append(payload.data(), payload.size()); + return value; +} + +io::CacheContext build_meta_cache_context() { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + return context; +} + +Status read_cached_range(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, size_t offset, + size_t size, const io::CacheContext& context, std::string* output) { + DCHECK(cache != nullptr); + DCHECK(output != nullptr); + output->clear(); + if (size == 0) { + return Status::OK(); + } + + io::FileBlocks blocks; + bool fully_covered = false; + RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, + &blocks, &fully_covered)); + if (!fully_covered) { + return Status::NotFound("file meta disk cache range is not cached"); + } + + output->resize(size); + size_t copied_size = 0; + const size_t requested_right = offset + size - 1; + for (const auto& block : blocks) { + const auto& range = block->range(); + const size_t read_left = std::max(range.left, offset); + const size_t read_right = std::min(range.right, requested_right); + DCHECK_LE(read_left, read_right); + const size_t read_size = read_right - read_left + 1; + RETURN_IF_ERROR(block->read(Slice(output->data() + copied_size, read_size), + read_left - range.left)); + copied_size += read_size; + } + DCHECK_EQ(copied_size, size); + return Status::OK(); +} + +} // namespace + +std::string FileMetaDiskCache::get_key(FileMetaCacheFormat format, + std::string_view file_meta_cache_key) { + std::string key; + key.reserve(32 + file_meta_cache_key.size()); + key.append("file_meta_cache:v1:"); + key.append(format_name(format)); + key.push_back(':'); + key.append(file_meta_cache_key.data(), file_meta_cache_key.size()); + return key; +} + +Status FileMetaDiskCache::read(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, std::string* payload) { + payload->clear(); + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + + auto invalidate_entry = [&](const Status& status) { + payload->clear(); + cache->remove_if_cached(hash); + return status; + }; + + std::string header; + Status status = + read_cached_range(cache, hash, 0, FILE_META_DISK_CACHE_HEADER_SIZE, context, &header); + if (!status.ok()) { + return status; + } + + FileMetaDiskCacheHeader parsed; + status = parse_header(header, &parsed); + if (!status.ok()) { + return invalidate_entry(status); + } + if (parsed.format != format || parsed.modification_time != modification_time || + parsed.file_size != file_size || + !FileMetaCache::is_persistent_cache_payload_size_allowed(parsed.payload_size)) { + return invalidate_entry(Status::NotFound("file meta disk cache header mismatch")); + } + + status = read_cached_range(cache, hash, FILE_META_DISK_CACHE_HEADER_SIZE, parsed.payload_size, + context, payload); + if (!status.ok()) { + return status; + } + const uint32_t checksum = crc32c::Crc32c(payload->data(), payload->size()); + if (checksum != parsed.checksum) { + return invalidate_entry(Status::NotFound("file meta disk cache checksum mismatch")); + } + return Status::OK(); +} + +Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, + std::string_view payload) { + if (!FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(payload.size()))) { + return Status::NotFound("file meta disk cache payload is too large"); + } + + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + + const std::string value = build_cache_value(format, modification_time, file_size, payload); + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + auto holder = cache->get_or_set(hash, 0, value.size(), context); + for (const auto& block : holder.file_blocks) { + auto state = block->state(); + if (state == io::FileBlock::State::DOWNLOADING && !block->is_downloader()) { + state = block->wait(); + } + if (state == io::FileBlock::State::DOWNLOADED) { + continue; + } + if (state != io::FileBlock::State::EMPTY) { + cache->remove_if_cached(hash); + return Status::NotFound("file meta disk cache block is not writable"); + } + + if (block->get_or_set_downloader() != io::FileBlock::get_caller_id()) { + cache->remove_if_cached(hash); + return Status::NotFound("file meta disk cache block has another downloader"); + } + const auto& range = block->range(); + DCHECK_LT(range.right, value.size()); + Status status = block->append(Slice(value.data() + range.left, range.size())); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + status = block->finalize(); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + } + return Status::OK(); +} + +void FileMetaDiskCache::remove(FileMetaCacheFormat format, const std::string& file_meta_cache_key) { + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache != nullptr) { + cache->remove_if_cached(hash); + } +} + +io::BlockFileCache* FileMetaDiskCache::get_cache(const io::UInt128Wrapper& hash) const { + if (_cache != nullptr) { + return _cache; + } + io::FileCacheFactory* factory = io::FileCacheFactory::instance(); + if (factory == nullptr || factory->get_cache_instance_size() == 0) { + return nullptr; + } + return factory->get_by_path(hash); +} + +} // namespace doris diff --git a/be/src/io/fs/file_meta_disk_cache.h b/be/src/io/fs/file_meta_disk_cache.h new file mode 100644 index 00000000000000..e844690764a3b0 --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "io/fs/file_meta_cache.h" + +namespace doris { + +namespace io { +class BlockFileCache; +struct UInt128Wrapper; +} // namespace io + +class FileMetaDiskCache final : public FileMetaPersistentCache { +public: + explicit FileMetaDiskCache(io::BlockFileCache* cache = nullptr) : _cache(cache) {} + + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override; + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override; + + void remove(FileMetaCacheFormat format, const std::string& key) override; + +private: + static std::string get_key(FileMetaCacheFormat format, std::string_view file_meta_cache_key); + + io::BlockFileCache* get_cache(const io::UInt128Wrapper& hash) const; + + io::BlockFileCache* _cache = nullptr; +}; + +} // namespace doris diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index c010c1deeba272..8b2b49a195bbc6 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -62,6 +62,7 @@ #include "io/cache/block_file_cache_factory.h" #include "io/cache/fs_file_cache_storage.h" #include "io/fs/file_meta_cache.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/local_file_reader.h" #include "load/channel/load_channel_mgr.h" #include "load/channel/load_stream_mgr.h" @@ -506,7 +507,9 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths "= true"; exit(-1); } - return; + if (!FileMetaCache::is_persistent_cache_enabled()) { + return; + } } if (config::file_cache_each_block_size > config::s3_write_buffer_size || config::s3_write_buffer_size % config::file_cache_each_block_size != 0) { @@ -676,7 +679,10 @@ Status ExecEnv::init_mem_env() { << config::file_cache_max_file_reader_cache_size; config::file_cache_max_file_reader_cache_size = block_file_cache_fd_cache_size; - _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num); + _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num, + FileMetaCache::is_persistent_cache_enabled() + ? std::make_unique() + : nullptr); _lookup_connection_cache = LookupConnectionCache::create_global_instance(config::lookup_connection_cache_capacity); diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 07eb171e9ebc5d..286ba083665776 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -22,6 +22,8 @@ #include "common/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "io/cache/block_file_cache.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/file_reader.h" #include "util/defer_op.h" @@ -226,4 +228,57 @@ TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); } +TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings; + settings.capacity = 1024 * 1024; + settings.max_file_block_size = 8; + settings.index_queue_size = 1024 * 1024; + settings.index_queue_elements = 1024; + settings.max_query_cache_size = 1024 * 1024; + settings.storage = "memory"; + io::BlockFileCache block_cache("file_meta_disk_cache_interface_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(1024 * 1024, std::move(disk_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/interface.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + const std::string payload = "serialized footer payload"; + auto value = std::make_unique(payload); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult insert_result = cache.insert(context, value, &handle, payload); + ASSERT_TRUE(insert_result.persisted_inserted); + ASSERT_FALSE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string output; + const FileMetaCacheLookupResult lookup_result = cache.lookup(context, &lookup_handle, &output); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(output, payload); + EXPECT_FALSE(lookup_handle.valid()); + + const FileMetaCacheContext stale_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 124, + .file_size = 456, + .enable_memory_cache = false}; + output.clear(); + const FileMetaCacheLookupResult stale_lookup_result = + cache.lookup(stale_context, &lookup_handle, &output); + EXPECT_EQ(stale_lookup_result.state, FileMetaCacheLookupState::MISS); + EXPECT_TRUE(output.empty()); +} + } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 6269a9fc2a7245..fa608e4289a2ac 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -60,6 +60,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -1073,7 +1074,8 @@ class NewParquetReaderTest : public testing::Test { int64_t range_start_offset = 0, int64_t range_size = -1, RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, - std::optional global_rowid_context = std::nullopt) const { + std::optional global_rowid_context = std::nullopt, + FileMetaCache* file_meta_cache = nullptr) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1083,7 +1085,7 @@ class NewParquetReaderTest : public testing::Test { file_description->range_size = range_size; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, file_meta_cache); } std::filesystem::path _test_dir; @@ -1108,6 +1110,28 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, UsesFileMetaCacheForFooterMetadata) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = + create_reader(0, -1, nullptr, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 0, file_size); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(file_meta_cache_key, &handle)); + EXPECT_TRUE(handle.valid()); + + auto second_reader = + create_reader(0, -1, nullptr, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(second_reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. From c0936422c74ca0b833ea1c59125155dcc44f6e88 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Thu, 9 Jul 2026 21:54:58 +0800 Subject: [PATCH 03/11] [fix](be) Fix external file meta disk cache correctness ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: The external file metadata disk cache integration needed stronger correctness boundaries before review. Memory cache keys did not include the metadata format, so legacy Parquet and v2 Parquet could collide while storing incompatible in-memory objects. Persistent entries could also be attempted for files without a stable modification time, and missing disk-cache ranges were treated the same as corrupt entries. This change scopes memory keys by format, adds a PARQUET_V2 persistent format, builds persistent cache identities from path, modification time, and file size, skips persistent cache when the file identity is unstable, handles missing ranges without invalidating concurrent writers, falls back to file reads on invalid persistent payloads, and separates memory-cache gating from persistent-cache gating in file scanners. ### Release note None ### Check List (For Author) - Test: Unit Test / Manual test - git diff --check - git diff --cached --check - ./run-be-ut.sh --run --filter='*FileMeta*' -j 2 - Manual e2e with output FE/BE: local Parquet TVF returned count 25 twice and second profile showed FileFooterHitMemoryCache=1; local ORC TVF returned count 25 twice and second profile showed FileFooterHitMemoryCache=1. Local TVF does not provide modification_time, so persistent disk cache is intentionally skipped on that path and covered by FileMetaCacheDiskTest. - Behavior changed: Yes. File metadata persistent cache now requires a stable modification time, uses format-scoped identities, and avoids invalidating incomplete in-flight cache writes. - Does this need documentation: No --- be/src/exec/scan/file_scanner.cpp | 21 ++ be/src/exec/scan/file_scanner.h | 12 +- be/src/exec/scan/file_scanner_v2.cpp | 9 +- be/src/exec/scan/file_scanner_v2.h | 1 + be/src/format/generic_reader.h | 3 + be/src/format/orc/vorc_reader.cpp | 15 +- be/src/format/parquet/vparquet_reader.cpp | 40 ++- be/src/format_v2/file_reader.h | 6 + .../parquet/parquet_file_context.cpp | 44 ++- .../format_v2/parquet/parquet_file_context.h | 3 +- be/src/format_v2/parquet/parquet_profile.cpp | 12 + be/src/format_v2/parquet/parquet_profile.h | 6 + be/src/format_v2/parquet/parquet_reader.cpp | 34 +- be/src/format_v2/parquet/parquet_reader.h | 4 +- be/src/format_v2/table/hudi_reader.cpp | 2 + be/src/format_v2/table/paimon_reader.cpp | 2 + be/src/format_v2/table_reader.cpp | 4 +- be/src/format_v2/table_reader.h | 2 + be/src/io/fs/file_meta_cache.cpp | 34 +- be/src/io/fs/file_meta_cache.h | 7 +- be/src/io/fs/file_meta_disk_cache.cpp | 61 +++- .../file_reader/file_meta_cache_test.cpp | 306 +++++++++++++++++- .../format_v2/parquet/parquet_reader_test.cpp | 231 ++++++++++++- 23 files changed, 764 insertions(+), 95 deletions(-) diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 78d6421adb5054..7c36b4daabe34a 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1346,6 +1346,10 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr parquet_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + const bool enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; phmap::flat_hash_map>> slot_id_to_predicates = _local_state @@ -1371,6 +1375,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&pctx); _cur_reader = std::move(iceberg_reader); } else if (range.__isset.table_format_params && @@ -1379,6 +1384,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _kv_cache, _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&pctx); _cur_reader = std::move(paimon_reader); } else if (range.__isset.table_format_params && @@ -1387,6 +1393,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&pctx); _cur_reader = std::move(hudi_reader); } else if (range.table_format_params.table_format_type == "hive") { @@ -1398,6 +1405,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&pctx); _cur_reader = std::move(hive_reader); } else if (range.table_format_params.table_format_type == "tvf") { @@ -1411,6 +1419,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } else if (_is_load) { @@ -1420,6 +1429,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, _io_ctx, _state, file_meta_cache_ptr, _state->query_options().enable_parquet_lazy_mat); } + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } @@ -1431,6 +1441,10 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr orc_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + const bool enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; // Build unified OrcInitContext (shared by all ORC reader variants) OrcInitContext octx; @@ -1449,6 +1463,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(tran_orc_reader.get()); init_status = static_cast(tran_orc_reader.get())->init_reader(&octx); _cur_reader = std::move(tran_orc_reader); @@ -1462,6 +1477,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&octx); _cur_reader = std::move(iceberg_reader); @@ -1471,6 +1487,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonOrcReader::create_unique( _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _kv_cache, _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&octx); _cur_reader = std::move(paimon_reader); @@ -1480,6 +1497,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiOrcReader::create_unique(_profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&octx); _cur_reader = std::move(hudi_reader); @@ -1493,6 +1511,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&octx); _cur_reader = std::move(hive_reader); @@ -1507,6 +1526,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } else if (_is_load) { @@ -1515,6 +1535,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr, _state->query_options().enable_orc_lazy_mat); } + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 2781771685f190..c7e124e0d447a7 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -326,13 +326,15 @@ class FileScanner : public Scanner { : _local_state->get_push_down_agg_type(); } - // enable the file meta cache only when - // 1. max_external_file_meta_cache_num is > 0 - // 2. the file number is less than 1/3 of cache's capacibility - // Otherwise, the cache miss rate will be high bool _should_enable_file_meta_cache() { return (ExecEnv::GetInstance()->file_meta_cache()->enabled() || - FileMetaCache::is_persistent_cache_enabled()) && + FileMetaCache::is_persistent_cache_enabled()); + } + + // Enable memory file meta cache only when the file number is less than 1/3 of cache capacity. + // Otherwise, the cache miss rate will be high. Persistent cache is not gated by this rule. + bool _should_enable_file_meta_memory_cache() { + return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 34b4eb758ec516..f98a06f91efcfb 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -455,6 +455,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .push_down_agg_type = _local_state->get_push_down_agg_type(), .condition_cache_digest = _local_state->get_condition_cache_digest(), .file_meta_cache = file_meta_cache, + .enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(), })); return Status::OK(); } @@ -518,8 +519,12 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not } bool FileScannerV2::_should_enable_file_meta_cache() const { - return (ExecEnv::GetInstance()->file_meta_cache()->enabled() || - FileMetaCache::is_persistent_cache_enabled()) && + return ExecEnv::GetInstance()->file_meta_cache()->enabled() || + FileMetaCache::is_persistent_cache_enabled(); +} + +bool FileScannerV2::_should_enable_file_meta_memory_cache() const { + return ExecEnv::GetInstance()->file_meta_cache()->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 66242b610f4373..8876ab6a6669eb 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -120,6 +120,7 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); bool _should_enable_file_meta_cache() const; + bool _should_enable_file_meta_memory_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; Status _generate_partition_values(const TFileRangeDesc& range, diff --git a/be/src/format/generic_reader.h b/be/src/format/generic_reader.h index cc21ce82c2805e..2f0589c4e6a235 100644 --- a/be/src/format/generic_reader.h +++ b/be/src/format/generic_reader.h @@ -258,6 +258,8 @@ class GenericReader : public ProfileCollector { // Pass condition cache context to the reader for HIT/MISS tracking. virtual void set_condition_cache_context(std::shared_ptr ctx) {} + void set_enable_file_meta_memory_cache(bool enable) { _enable_file_meta_memory_cache = enable; } + // Returns true if this reader can produce an accurate total row count from metadata // without reading actual data. Used to determine if CountReader decorator can be applied. // Only ORC and Parquet readers support this (via file footer metadata). @@ -279,6 +281,7 @@ class GenericReader : public ProfileCollector { // Cache to save some common part such as file footer. // Maybe null if not used FileMetaCache* _meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; // ---- Column descriptors (set by init_reader, owned by FileScanner) ---- const std::vector* _column_descs = nullptr; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index c4907bc3961072..09dfd720476efb 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -472,7 +472,7 @@ Status OrcReader::_create_file_reader() { .key = file_meta_cache_key, .modification_time = _file_description.mtime, .file_size = file_size, - .enable_memory_cache = _meta_cache->enabled()}; + .enable_memory_cache = _enable_file_meta_memory_cache && _meta_cache->enabled()}; FileMetaCacheProfile file_meta_cache_profile { .hit_cache = &_statistics.file_footer_hit_cache, .hit_memory_cache = &_statistics.file_footer_hit_memory_cache, @@ -481,6 +481,8 @@ Status OrcReader::_create_file_reader() { .write_disk_cache = &_statistics.file_footer_write_disk_cache, .read_disk_cache_time = &_statistics.file_footer_read_disk_cache_time, .write_disk_cache_time = &_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); // Local variables can be required because setSerializedFileTail is an assignment operation, not a reference. ObjLRUCache::CacheHandle _meta_cache_handle; @@ -495,9 +497,16 @@ Status OrcReader::_create_file_reader() { } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { auto footer_ptr = std::make_unique(std::move(serialized_file_tail)); options.setSerializedFileTail(*footer_ptr); - RETURN_IF_ERROR(create_orc_reader()); + Status status = create_orc_reader(); + if (!status.ok()) { + VLOG_DEBUG << "ignore invalid orc file meta persistent cache payload: " << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _reader.reset(); + _file_input_stream.reset(); + return _create_file_reader(); + } if (file_meta_cache_context.enable_memory_cache) { - _meta_cache->insert(file_meta_cache_key, footer_ptr, &_meta_cache_handle); + _meta_cache->insert(memory_cache_key, footer_ptr, &_meta_cache_handle); } } else { _statistics.file_footer_read_calls++; diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 8dc3dd7154ebcc..09ca47da5853c4 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -390,7 +390,8 @@ Status ParquetReader::_open_file() { .key = file_meta_cache_key, .modification_time = _file_description.mtime, .file_size = file_size, - .enable_memory_cache = _meta_cache->enabled()}; + .enable_memory_cache = + _enable_file_meta_memory_cache && _meta_cache->enabled()}; FileMetaCacheProfile file_meta_cache_profile { .hit_cache = &_reader_statistics.file_footer_hit_cache, .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, @@ -399,6 +400,8 @@ Status ParquetReader::_open_file() { .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); std::string footer_payload; const FileMetaCacheLookupResult lookup_result = _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, @@ -408,20 +411,31 @@ Status ParquetReader::_open_file() { } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { uint32_t metadata_size = static_cast(footer_payload.size()); tparquet::FileMetaData t_metadata; - RETURN_IF_ERROR(deserialize_thrift_msg( + Status status = deserialize_thrift_msg( reinterpret_cast(footer_payload.data()), &metadata_size, - true, &t_metadata)); - _file_metadata_ptr = std::make_unique(t_metadata, metadata_size); - RETURN_IF_ERROR(_file_metadata_ptr->init_schema(enable_mapping_varbinary, - enable_mapping_timestamp_tz)); - if (file_meta_cache_context.enable_memory_cache) { - _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), - &_meta_cache_handle); - _file_metadata = _meta_cache_handle.data(); + true, &t_metadata); + if (status.ok()) { + _file_metadata_ptr = std::make_unique(t_metadata, metadata_size); + status = _file_metadata_ptr->init_schema(enable_mapping_varbinary, + enable_mapping_timestamp_tz); + } + if (status.ok()) { + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(memory_cache_key, _file_metadata_ptr.release(), + &_meta_cache_handle); + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } } else { - _file_metadata = _file_metadata_ptr.get(); + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _file_metadata_ptr.reset(); + footer_payload.clear(); } - } else { + } + if (_file_metadata == nullptr) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, enable_mapping_timestamp_tz)); @@ -442,7 +456,7 @@ Status ParquetReader::_open_file() { _file_metadata = _file_metadata_ptr.get(); } } else if (file_meta_cache_context.enable_memory_cache && - _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr, + _meta_cache->insert(memory_cache_key, _file_metadata_ptr, &_meta_cache_handle)) { _file_metadata = _meta_cache_handle.data(); } else { diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 135df2820145e3..522f10abbe2848 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -227,6 +227,12 @@ class FileReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 81af94878ca155..caf8c2f6d2db5e 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -32,6 +32,7 @@ #include "common/check.h" #include "common/config.h" +#include "common/logging.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" @@ -537,7 +538,8 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont bool enable_page_cache, const io::FileDescription& file_description, FileMetaCache* file_meta_cache, FileMetaCacheProfile* file_meta_cache_profile, - int64_t* file_footer_read_calls) { + int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache) { DORIS_CHECK(input_file_reader != nullptr); DORIS_CHECK(file_footer_read_calls != nullptr); auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); @@ -550,11 +552,16 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont std::shared_ptr<::parquet::FileMetaData> cached_metadata; bool metadata_from_cache = false; FileMetaCacheContext file_meta_cache_context { - .format = FileMetaCacheFormat::PARQUET, + .format = FileMetaCacheFormat::PARQUET_V2, .key = file_meta_cache_key, .modification_time = file_description.mtime, .file_size = file_size, - .enable_memory_cache = file_meta_cache != nullptr && file_meta_cache->enabled()}; + .enable_memory_cache = enable_file_meta_memory_cache && file_meta_cache != nullptr && + file_meta_cache->enabled()}; + const std::string memory_cache_key = + file_meta_cache != nullptr + ? FileMetaCache::get_memory_cache_key(file_meta_cache_context) + : std::string {}; const auto reader_properties = ::parquet::default_reader_properties(); try { if (file_meta_cache != nullptr) { @@ -570,18 +577,25 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont cached_metadata = *cached_metadata_ptr; metadata_from_cache = true; } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { - uint32_t metadata_len = static_cast(serialized_metadata.size()); - cached_metadata = ::parquet::FileMetaData::Make(serialized_metadata.data(), - &metadata_len, reader_properties); - if (file_meta_cache_context.enable_memory_cache) { - auto cached_metadata_holder = - std::make_unique>( - cached_metadata); - ObjLRUCache::CacheHandle insert_handle; - file_meta_cache->insert(file_meta_cache_key, cached_metadata_holder, - &insert_handle); + try { + uint32_t metadata_len = static_cast(serialized_metadata.size()); + cached_metadata = ::parquet::FileMetaData::Make( + serialized_metadata.data(), &metadata_len, reader_properties); + if (file_meta_cache_context.enable_memory_cache) { + auto cached_metadata_holder = + std::make_unique>( + cached_metadata); + ObjLRUCache::CacheHandle insert_handle; + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, + &insert_handle); + } + metadata_from_cache = true; + } catch (const std::exception& e) { + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << e.what(); + file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + cached_metadata.reset(); } - metadata_from_cache = true; } } arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, @@ -603,7 +617,7 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont &insert_handle, serialized_metadata, file_meta_cache_profile); } else if (file_meta_cache_context.enable_memory_cache) { - file_meta_cache->insert(file_meta_cache_key, cached_metadata_holder, + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, &insert_handle); } } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 59ae5a9d0ce057..00e5277b8811bf 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -115,7 +115,8 @@ struct ParquetFileContext { Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, FileMetaCache* file_meta_cache, - FileMetaCacheProfile* file_meta_cache_profile, int64_t* file_footer_read_calls); + FileMetaCacheProfile* file_meta_cache_profile, int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache = true); // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page // decoding, so v2 caches the serialized bytes read inside these ranges and excludes // footer/metadata reads that happen before registration. diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index d41ff295ec4b3e..3ee99db7619824 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -103,6 +103,18 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 27f9818f4a0dcd..8e5edba1b1ba60 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -116,6 +116,12 @@ struct ParquetProfile { RuntimeProfile::Counter* open_file_num = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* row_group_filter_time = nullptr; RuntimeProfile::Counter* page_index_read_calls = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index bc70d943f470d7..384f04c946c5da 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -313,11 +313,13 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), - _file_meta_cache(file_meta_cache) {} + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} ParquetReader::~ParquetReader() = default; @@ -351,16 +353,34 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_batch_size(_batch_size); // Open parquet file and parse metadata to get file schema. FileMetaCacheProfile file_meta_cache_profile { - .hit_cache = &_reader_statistics.file_footer_hit_cache}; - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description, - _file_meta_cache, &file_meta_cache_profile, - &_reader_statistics.file_footer_read_calls)); + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _file_meta_cache, &file_meta_cache_profile, &_reader_statistics.file_footer_read_calls, + _enable_file_meta_memory_cache)); if (_profile != nullptr) { COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, _reader_statistics.file_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 8f88e2f5fb92b6..654b8850cd26f3 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -48,7 +48,8 @@ class ParquetReader : public format::FileReader { std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, bool enable_mapping_timestamp_tz = false, - FileMetaCache* file_meta_cache = nullptr); + FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -92,6 +93,7 @@ class ParquetReader : public format::FileReader { size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 3600b0ff706f08..961757c16a5776 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -126,6 +126,8 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, }); } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index c2114968946e1b..9c0ca96ec70a12 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -154,6 +154,8 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, }); } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 05b3d4d8117175..778ce39a6409f7 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -471,6 +471,7 @@ Status TableReader::init(TableReadOptions&& options) { _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; _file_meta_cache = options.file_meta_cache; + _enable_file_meta_memory_cache = options.enable_file_meta_memory_cache; _file_slot_descs = options.file_slot_descs; _push_down_agg_type = options.push_down_agg_type; _condition_cache_digest = options.condition_cache_digest; @@ -678,7 +679,8 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { if (_format == FileFormat::PARQUET) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::ORC) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 37ff0011530568..cb6416e1c60ce9 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -139,6 +139,7 @@ struct TableReadOptions { // Digest of stable pushed-down predicates. A zero digest disables condition cache. uint64_t condition_cache_digest = 0; FileMetaCache* file_meta_cache = nullptr; + bool enable_file_meta_memory_cache = true; }; struct SplitReadOptions { @@ -1528,6 +1529,7 @@ class TableReader { RuntimeState* _runtime_state; RuntimeProfile* _scanner_profile; FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index db5e04f63ec44f..a9c65dfce3f356 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -32,6 +32,10 @@ void update_profile_counter(int64_t* counter, int64_t value = 1) { } } +bool is_persistent_cache_identity_stable(const FileMetaCacheContext& context) { + return context.modification_time > 0; +} + } // namespace FileMetaCache::FileMetaCache(int64_t capacity, @@ -41,14 +45,11 @@ FileMetaCache::FileMetaCache(int64_t capacity, std::string FileMetaCache::get_key(const std::string file_name, int64_t modification_time, int64_t file_size) { std::string meta_cache_key; - meta_cache_key.resize(file_name.size() + sizeof(int64_t)); + meta_cache_key.resize(file_name.size() + 2 * sizeof(int64_t)); memcpy(meta_cache_key.data(), file_name.data(), file_name.size()); - if (modification_time != 0) { - memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); - } else { - memcpy(meta_cache_key.data() + file_name.size(), &file_size, sizeof(int64_t)); - } + memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); + memcpy(meta_cache_key.data() + file_name.size() + sizeof(int64_t), &file_size, sizeof(int64_t)); return meta_cache_key; } @@ -59,6 +60,14 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } +std::string FileMetaCache::get_memory_cache_key(FileMetaCacheFormat format, std::string_view key) { + std::string memory_cache_key; + memory_cache_key.reserve(key.size() + sizeof(uint8_t)); + memory_cache_key.push_back(static_cast(format)); + memory_cache_key.append(key.data(), key.size()); + return memory_cache_key; +} + bool FileMetaCache::is_persistent_cache_enabled() { return config::enable_external_file_meta_disk_cache && config::external_file_meta_disk_cache_max_entry_bytes > 0; @@ -67,7 +76,7 @@ bool FileMetaCache::is_persistent_cache_enabled() { bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) { const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 && - payload_size <= static_cast(max_entry_bytes); + payload_size > 0 && payload_size <= static_cast(max_entry_bytes); } FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context, @@ -76,7 +85,7 @@ FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& cont FileMetaCacheProfile* profile) { DCHECK(handle != nullptr); DCHECK(serialized_meta != nullptr); - if (context.enable_memory_cache && lookup(context.key, handle)) { + if (context.enable_memory_cache && lookup(get_memory_cache_key(context), handle)) { serialized_meta->clear(); if (profile != nullptr) { update_profile_counter(profile->hit_cache); @@ -94,8 +103,10 @@ FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& cont update_profile_counter(profile->hit_disk_cache); update_profile_counter(profile->read_disk_cache_time, persisted_read_time); } - } else if (is_persistent_cache_enabled() && profile != nullptr) { + } else if (is_persistent_cache_enabled() && is_persistent_cache_identity_stable(context) && + profile != nullptr) { update_profile_counter(profile->miss_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); } return result; } @@ -112,7 +123,8 @@ bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context, DCHECK(read_time != nullptr); payload->clear(); *read_time = 0; - if (!is_persistent_cache_enabled() || _persistent_cache == nullptr) { + if (!is_persistent_cache_enabled() || _persistent_cache == nullptr || + !is_persistent_cache_identity_stable(context)) { return false; } @@ -134,7 +146,7 @@ bool FileMetaCache::insert_persistent_cache(const FileMetaCacheContext& context, DCHECK(write_time != nullptr); *write_time = 0; if (!is_persistent_cache_payload_size_allowed(static_cast(payload.size())) || - _persistent_cache == nullptr) { + _persistent_cache == nullptr || !is_persistent_cache_identity_stable(context)) { return false; } diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index a7b18f94b924c6..3092d61479092e 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -32,6 +32,7 @@ namespace doris { enum class FileMetaCacheFormat : uint8_t { PARQUET = 1, ORC = 2, + PARQUET_V2 = 3, }; struct FileMetaCacheContext { @@ -99,6 +100,10 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); + static std::string get_memory_cache_key(FileMetaCacheFormat format, std::string_view key); + static std::string get_memory_cache_key(const FileMetaCacheContext& context) { + return get_memory_cache_key(context.format, context.key); + } static bool is_persistent_cache_enabled(); static bool is_persistent_cache_payload_size_allowed(uint64_t payload_size); @@ -149,7 +154,7 @@ class FileMetaCache { } } if (context.enable_memory_cache) { - result.memory_inserted = insert(context.key, value, handle); + result.memory_inserted = insert(get_memory_cache_key(context), value, handle); } return result; } diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp index d20316917e3700..ba7e1a5b4fcae1 100644 --- a/be/src/io/fs/file_meta_disk_cache.cpp +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -46,18 +46,19 @@ std::string_view format_name(FileMetaCacheFormat format) { return "parquet"; case FileMetaCacheFormat::ORC: return "orc"; + case FileMetaCacheFormat::PARQUET_V2: + return "parquet_v2"; } DCHECK(false) << "unknown file meta cache format"; return "unknown"; } Status parse_format(uint8_t format, FileMetaCacheFormat* parsed) { - if (format == static_cast(FileMetaCacheFormat::PARQUET)) { - *parsed = FileMetaCacheFormat::PARQUET; - return Status::OK(); - } - if (format == static_cast(FileMetaCacheFormat::ORC)) { - *parsed = FileMetaCacheFormat::ORC; + switch (static_cast(format)) { + case FileMetaCacheFormat::PARQUET: + case FileMetaCacheFormat::ORC: + case FileMetaCacheFormat::PARQUET_V2: + *parsed = static_cast(format); return Status::OK(); } return Status::NotFound("file meta disk cache format mismatch"); @@ -125,10 +126,14 @@ io::CacheContext build_meta_cache_context() { } Status read_cached_range(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, size_t offset, - size_t size, const io::CacheContext& context, std::string* output) { + size_t size, const io::CacheContext& context, std::string* output, + bool* missing_cached_range = nullptr) { DCHECK(cache != nullptr); DCHECK(output != nullptr); output->clear(); + if (missing_cached_range != nullptr) { + *missing_cached_range = false; + } if (size == 0) { return Status::OK(); } @@ -138,6 +143,9 @@ Status read_cached_range(io::BlockFileCache* cache, const io::UInt128Wrapper& ha RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, &blocks, &fully_covered)); if (!fully_covered) { + if (missing_cached_range != nullptr) { + *missing_cached_range = true; + } return Status::NotFound("file meta disk cache range is not cached"); } @@ -192,10 +200,14 @@ Status FileMetaDiskCache::read(FileMetaCacheFormat format, const std::string& fi }; std::string header; - Status status = - read_cached_range(cache, hash, 0, FILE_META_DISK_CACHE_HEADER_SIZE, context, &header); + bool missing_header_range = false; + Status status = read_cached_range(cache, hash, 0, FILE_META_DISK_CACHE_HEADER_SIZE, context, + &header, &missing_header_range); if (!status.ok()) { - return status; + if (missing_header_range) { + return status; + } + return invalidate_entry(status); } FileMetaDiskCacheHeader parsed; @@ -209,10 +221,15 @@ Status FileMetaDiskCache::read(FileMetaCacheFormat format, const std::string& fi return invalidate_entry(Status::NotFound("file meta disk cache header mismatch")); } + bool missing_payload_range = false; status = read_cached_range(cache, hash, FILE_META_DISK_CACHE_HEADER_SIZE, parsed.payload_size, - context, payload); + context, payload, &missing_payload_range); if (!status.ok()) { - return status; + if (missing_payload_range) { + payload->clear(); + return status; + } + return invalidate_entry(status); } const uint32_t checksum = crc32c::Crc32c(payload->data(), payload->size()); if (checksum != parsed.checksum) { @@ -226,7 +243,7 @@ Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& f std::string_view payload) { if (!FileMetaCache::is_persistent_cache_payload_size_allowed( static_cast(payload.size()))) { - return Status::NotFound("file meta disk cache payload is too large"); + return Status::NotFound("file meta disk cache payload size is not allowed"); } const std::string cache_key = get_key(format, file_meta_cache_key); @@ -241,21 +258,19 @@ Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& f io::CacheContext context = build_meta_cache_context(); context.stats = &stats; auto holder = cache->get_or_set(hash, 0, value.size(), context); - for (const auto& block : holder.file_blocks) { + auto write_block = [&](const io::FileBlockSPtr& block) -> Status { auto state = block->state(); if (state == io::FileBlock::State::DOWNLOADING && !block->is_downloader()) { state = block->wait(); } if (state == io::FileBlock::State::DOWNLOADED) { - continue; + return Status::OK(); } if (state != io::FileBlock::State::EMPTY) { - cache->remove_if_cached(hash); return Status::NotFound("file meta disk cache block is not writable"); } if (block->get_or_set_downloader() != io::FileBlock::get_caller_id()) { - cache->remove_if_cached(hash); return Status::NotFound("file meta disk cache block has another downloader"); } const auto& range = block->range(); @@ -270,6 +285,18 @@ Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& f cache->remove_if_cached(hash); return status; } + return Status::OK(); + }; + + for (const auto& block : holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_HEADER_SIZE) { + RETURN_IF_ERROR(write_block(block)); + } + } + for (const auto& block : holder.file_blocks) { + if (block->range().left < FILE_META_DISK_CACHE_HEADER_SIZE) { + RETURN_IF_ERROR(write_block(block)); + } } return Status::OK(); } diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 286ba083665776..17cb7686ea98ff 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -17,14 +17,19 @@ #include "io/fs/file_meta_cache.h" +#include + #include #include "common/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "io/cache/block_file_cache.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" #include "io/fs/file_meta_disk_cache.h" #include "io/fs/file_reader.h" +#include "util/coding.h" #include "util/defer_op.h" namespace doris { @@ -64,6 +69,64 @@ class MockFileReader : public io::FileReader { size_t _size; bool _closed; }; + +io::FileCacheSettings create_file_meta_disk_cache_test_settings() { + io::FileCacheSettings settings; + settings.capacity = 1024 * 1024; + settings.max_file_block_size = 8; + settings.index_queue_size = 1024 * 1024; + settings.index_queue_elements = 1024; + settings.max_query_cache_size = 1024 * 1024; + settings.storage = "memory"; + return settings; +} + +constexpr size_t FILE_META_DISK_CACHE_TEST_HEADER_SIZE = 36; + +std::string file_meta_disk_cache_key(FileMetaCacheFormat format, const std::string& meta_key) { + std::string key = "file_meta_cache:v1:"; + switch (format) { + case FileMetaCacheFormat::PARQUET: + key.append("parquet"); + break; + case FileMetaCacheFormat::ORC: + key.append("orc"); + break; + case FileMetaCacheFormat::PARQUET_V2: + key.append("parquet_v2"); + break; + } + key.push_back(':'); + key.append(meta_key); + return key; +} + +std::string build_file_meta_disk_cache_value(FileMetaCacheFormat format, int64_t modification_time, + int64_t file_size, std::string_view payload) { + std::string value; + value.append("DFMC", 4); + value.push_back(1); + value.push_back(static_cast(format)); + value.push_back(0); + value.push_back(0); + put_fixed64_le(&value, static_cast(file_size)); + put_fixed64_le(&value, static_cast(modification_time)); + put_fixed64_le(&value, payload.size()); + put_fixed32_le(&value, crc32c::Crc32c(payload.data(), payload.size())); + value.append(payload); + return value; +} + +io::CacheContext create_file_meta_disk_cache_context(io::ReadStatistics* stats) { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + context.stats = stats; + return context; +} } // anonymous namespace TEST(FileMetaCacheTest, KeyGenerationFromParams) { @@ -79,7 +142,11 @@ TEST(FileMetaCacheTest, KeyGenerationFromParams) { std::string key3 = FileMetaCache::get_key(file_name, mtime + 1, file_size); EXPECT_NE(key1, key3); - // mtime == 0, use file_size + // Same mtime with different size should produce different key + std::string key_with_different_size = FileMetaCache::get_key(file_name, mtime, file_size + 1); + EXPECT_NE(key1, key_with_different_size); + + // mtime == 0, still includes file_size std::string key4 = FileMetaCache::get_key(file_name, 0, file_size); std::string key5 = FileMetaCache::get_key(file_name, 0, file_size); EXPECT_EQ(key4, key5); @@ -122,19 +189,27 @@ TEST(FileMetaCacheTest, KeyContentVerification) { std::string key_with_mtime = FileMetaCache::get_key(file_name, mtime, file_size); - ASSERT_EQ(key_with_mtime.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_mtime.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_mtime.data(), file_name.data(), file_name.size()), 0); int64_t extracted_mtime = 0; memcpy(&extracted_mtime, key_with_mtime.data() + file_name.size(), sizeof(int64_t)); EXPECT_EQ(extracted_mtime, mtime); + int64_t extracted_file_size = 0; + memcpy(&extracted_file_size, key_with_mtime.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); + EXPECT_EQ(extracted_file_size, file_size); std::string key_with_filesize = FileMetaCache::get_key(file_name, 0, file_size); - ASSERT_EQ(key_with_filesize.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_filesize.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_filesize.data(), file_name.data(), file_name.size()), 0); + int64_t extracted_zero_mtime = -1; + memcpy(&extracted_zero_mtime, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + EXPECT_EQ(extracted_zero_mtime, 0); int64_t extracted_filesize = 0; - memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); EXPECT_EQ(extracted_filesize, file_size); } @@ -171,7 +246,7 @@ TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { auto value = std::make_unique("serialized footer payload"); ObjLRUCache::CacheHandle insert_handle; - ASSERT_TRUE(cache.insert(meta_key, value, &insert_handle)); + ASSERT_TRUE(cache.insert(FileMetaCache::get_memory_cache_key(context), value, &insert_handle)); ASSERT_TRUE(insert_handle.valid()); int64_t hit_cache = 0; @@ -197,6 +272,34 @@ TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { EXPECT_EQ(miss_disk_cache, 0); } +TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.parquet", 123, 456); + const FileMetaCacheContext parquet_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + const FileMetaCacheContext parquet_v2_context {.format = FileMetaCacheFormat::PARQUET_V2, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + + auto value = std::make_unique(7); + ObjLRUCache::CacheHandle insert_handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(parquet_context, value, &insert_handle, "payload"); + ASSERT_TRUE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + EXPECT_EQ(cache.lookup(parquet_v2_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(cache.lookup(parquet_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); +} + TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; @@ -228,6 +331,43 @@ TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); } +TEST(FileMetaCacheDiskTest, PersistentCacheRequiresStableModificationTime) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_unstable_identity_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/mtime-zero.parquet", 0, 456); + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(0, std::move(disk_cache)); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 0, + .file_size = 456, + .enable_memory_cache = false}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(context, value, &handle, std::string_view(*value)); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache}; + std::string serialized_meta; + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_EQ(cache.lookup(context, &lookup_handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 0); +} + TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; @@ -236,13 +376,7 @@ TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) }}; config::enable_external_file_meta_disk_cache = true; - io::FileCacheSettings settings; - settings.capacity = 1024 * 1024; - settings.max_file_block_size = 8; - settings.index_queue_size = 1024 * 1024; - settings.index_queue_elements = 1024; - settings.max_query_cache_size = 1024 * 1024; - settings.storage = "memory"; + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); io::BlockFileCache block_cache("file_meta_disk_cache_interface_test", settings); ASSERT_TRUE(block_cache.initialize().ok()); @@ -281,4 +415,152 @@ TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) EXPECT_TRUE(output.empty()); } +TEST(FileMetaCacheDiskTest, IncompleteMultiBlockEntryMissesWithoutInvalidating) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_incomplete_entry_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/incomplete.parquet", 123, 456); + const std::string payload = "serialized footer payload spanning several blocks"; + const std::string value = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto holder = block_cache.get_or_set(hash, 0, value.size(), context); + bool covered_header = false; + for (const auto& block : holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + continue; + } + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + covered_header = block->range().right + 1 >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE; + } + ASSERT_TRUE(covered_header); + } + + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + const Status status = + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(output.empty()); + EXPECT_FALSE(block_cache.get_blocks_by_key(hash).empty()); +} + +TEST(FileMetaCacheDiskTest, PayloadMissDoesNotRemoveInFlightWriterBlocks) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_inflight_writer_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/inflight.parquet", 123, 456); + const std::string payload = "serialized footer payload spanning several blocks"; + const std::string value = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto partial_holder = block_cache.get_or_set(hash, 0, value.size(), context); + for (const auto& block : partial_holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + continue; + } + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + } + } + + FileMetaDiskCache disk_cache(&block_cache); + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto writer_holder = block_cache.get_or_set(hash, 0, value.size(), context); + bool has_inflight_payload_block = false; + for (const auto& block : writer_holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + has_inflight_payload_block = true; + break; + } + } + ASSERT_TRUE(has_inflight_payload_block); + + std::string output; + const Status read_status = + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output); + EXPECT_FALSE(read_status.ok()); + EXPECT_TRUE(output.empty()); + + for (const auto& block : writer_holder.file_blocks) { + auto state = block->state(); + if (state == io::FileBlock::State::DOWNLOADED) { + continue; + } + if (state == io::FileBlock::State::EMPTY) { + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + } else { + ASSERT_EQ(state, io::FileBlock::State::DOWNLOADING); + ASSERT_TRUE(block->is_downloader()); + } + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + } + } + + std::string output; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); +} + +TEST(FileMetaCacheDiskTest, EmptyPayloadIsNotPersisted) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_empty_payload_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/empty.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, ""); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(block_cache.get_blocks_by_key(hash).empty()); +} + } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index fa608e4289a2ac..572d0b724c9810 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -35,6 +35,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_nullable.h" @@ -67,6 +68,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -1021,6 +1023,66 @@ class TestFileReader final : public format::FileReader { long io_context_use_count() const { return _io_ctx.use_count(); } }; +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + std::string result; + result.reserve(key.size() + 4); + result.append(std::to_string(static_cast(format))); + result.push_back(':'); + result.append(key); + return result; + } + + std::map _entries; + int _remove_count = 0; +}; + TEST(FileReaderTest, OpenStoresRequestAndCloseKeepsRequest) { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -1075,17 +1137,20 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - FileMetaCache* file_meta_cache = nullptr) const { + FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + file_description->mtime = 123; file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz, file_meta_cache); + global_rowid_context, enable_mapping_timestamp_tz, file_meta_cache, + enable_file_meta_memory_cache); } std::filesystem::path _test_dir; @@ -1114,24 +1179,178 @@ TEST_F(NewParquetReaderTest, UsesFileMetaCacheForFooterMetadata) { FileMetaCache file_meta_cache(1024 * 1024); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("new_parquet_reader_file_meta_cache_first"); auto first_reader = - create_reader(0, -1, nullptr, false, nullptr, std::nullopt, &file_meta_cache); + create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, &file_meta_cache); ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); const auto file_size = static_cast(std::filesystem::file_size(_file_path)); - const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 0, file_size); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); ObjLRUCache::CacheHandle handle; - EXPECT_TRUE(file_meta_cache.lookup(file_meta_cache_key, &handle)); + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); EXPECT_TRUE(handle.valid()); + RuntimeProfile second_profile("new_parquet_reader_file_meta_cache_second"); auto second_reader = - create_reader(0, -1, nullptr, false, nullptr, std::nullopt, &file_meta_cache); + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, &file_meta_cache); ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 0); std::vector schema; ASSERT_TRUE(second_reader->get_schema(&schema).ok()); ASSERT_EQ(schema.size(), 2); } +TEST_F(NewParquetReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(0, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_persistent_file_meta_cache_first"); + auto first_reader = + create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitDiskCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterMissDiskCache")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_persistent_file_meta_cache_second"); + auto second_reader = + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterMissDiskCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterWriteDiskCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key, 123, + file_size, "not a serialized parquet footer") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("new_parquet_reader_invalid_persistent_file_meta_cache_payload"); + auto reader = create_reader(0, -1, &profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE( + persistent_cache_ptr->contains(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key), + "not a serialized parquet footer"); +} + +TEST_F(NewParquetReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_skip_memory_file_meta_cache_first"); + auto first_reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, + &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_skip_memory_file_meta_cache_second"); + auto second_reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, + &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. From 9b946bc834ba54897e579a0bd9a2b11b2b74a389 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Fri, 10 Jul 2026 01:43:16 +0800 Subject: [PATCH 04/11] [fix](be) Guard file meta cache scanner toggles ### What problem does this PR solve? Issue Number: None Related PR: #63376 Problem Summary: BE UT crashed in OrcReadLinesTest.test0 because read_lines_from_range can initialize readers without an ExecEnv file meta cache object. The scanner memory-cache toggle still dereferenced ExecEnv::file_meta_cache()->enabled() directly. Use the actual FileMetaCache pointer selected for the reader when enabling memory cache and require an initialized cache object before enabling memory or persistent file meta cache. ### Release note None ### Check List (For Author) - Test: - Unit Test: ./run-be-ut.sh --run --filter='OrcReadLinesTest.test0' -j 1 - Unit Test: ./run-be-ut.sh --run --filter='OrcReadLinesTest.*:ParquetReadLinesTest.*:*FileMeta*' -j 1 - Manual test: build-support/clang-format.sh be/src/exec/scan/file_scanner.h be/src/exec/scan/file_scanner.cpp be/src/exec/scan/file_scanner_v2.h be/src/exec/scan/file_scanner_v2.cpp - Manual test: build-support/check-format.sh - Manual test: git diff --check - Manual test: git diff --cached --check - Behavior changed: No - Does this need documentation: No --- be/src/exec/scan/file_scanner.cpp | 6 ++++-- be/src/exec/scan/file_scanner.h | 9 +++++---- be/src/exec/scan/file_scanner_v2.cpp | 11 ++++++----- be/src/exec/scan/file_scanner_v2.h | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 7c36b4daabe34a..ce6dc5acd4695d 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1346,7 +1346,8 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr parquet_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); - const bool enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(); + const bool enable_file_meta_memory_cache = + _should_enable_file_meta_memory_cache(file_meta_cache_ptr); auto configure_file_meta_memory_cache = [&](GenericReader* reader) { reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); }; @@ -1441,7 +1442,8 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr orc_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); - const bool enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(); + const bool enable_file_meta_memory_cache = + _should_enable_file_meta_memory_cache(file_meta_cache_ptr); auto configure_file_meta_memory_cache = [&](GenericReader* reader) { reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); }; diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index c7e124e0d447a7..2fd6e00ac21e21 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -327,14 +327,15 @@ class FileScanner : public Scanner { } bool _should_enable_file_meta_cache() { - return (ExecEnv::GetInstance()->file_meta_cache()->enabled() || - FileMetaCache::is_persistent_cache_enabled()); + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); } // Enable memory file meta cache only when the file number is less than 1/3 of cache capacity. // Otherwise, the cache miss rate will be high. Persistent cache is not gated by this rule. - bool _should_enable_file_meta_memory_cache() { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index f98a06f91efcfb..953ebd2351da63 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -455,7 +455,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .push_down_agg_type = _local_state->get_push_down_agg_type(), .condition_cache_digest = _local_state->get_condition_cache_digest(), .file_meta_cache = file_meta_cache, - .enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(), + .enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(file_meta_cache), })); return Status::OK(); } @@ -519,12 +519,13 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not } bool FileScannerV2::_should_enable_file_meta_cache() const { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() || - FileMetaCache::is_persistent_cache_enabled(); + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); } -bool FileScannerV2::_should_enable_file_meta_memory_cache() const { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && +bool FileScannerV2::_should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 8876ab6a6669eb..d4f067bae3910b 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -120,7 +120,7 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); bool _should_enable_file_meta_cache() const; - bool _should_enable_file_meta_memory_cache() const; + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; Status _generate_partition_values(const TFileRangeDesc& range, From 72e490b26ab2e309c1e0d1be648b6e6195665252 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Fri, 10 Jul 2026 11:13:35 +0800 Subject: [PATCH 05/11] [fix](be) Fix file meta cache pipeline failures ### What problem does this PR solve? Issue Number: N/A Related PR: #64961 Problem Summary: Row-id fetch scanners read a mapped range without a split source, but Parquet and ORC reader initialization evaluated the normal scan-range memory file metadata cache admission rule and dereferenced the missing split source. This crashed external, P0, and cloud regression jobs. The unknown-mtime Parquet unit test also accidentally used a synthetic nonzero mtime, enabling page cache while asserting it remained disabled. Disable memory file metadata cache for row-id fetch reader initialization and make the unit test pass an explicit unknown mtime. ### Release note None ### Check List (For Author) - Test: Unit Test / Code style check - `./run-be-ut.sh --run --filter='OrcReadLinesTest.*:ParquetReadLinesTest.*:NewParquetReaderTest.RewriteSameLocalPathDoesNotReuseUnknownMtimePageCache' -j 1` - `build-support/check-format.sh` - `git diff --check` - `git diff --cached --check` - Behavior changed: No - Does this need documentation: No --- be/src/exec/scan/file_scanner.cpp | 6 ++++-- be/test/format_v2/parquet/parquet_reader_test.cpp | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index ce6dc5acd4695d..5b9bf67eacf490 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1346,8 +1346,9 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr parquet_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. const bool enable_file_meta_memory_cache = - _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); auto configure_file_meta_memory_cache = [&](GenericReader* reader) { reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); }; @@ -1442,8 +1443,9 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr orc_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. const bool enable_file_meta_memory_cache = - _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); auto configure_file_meta_memory_cache = [&](GenericReader* reader) { reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); }; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 572d0b724c9810..1b22419a80322c 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1137,14 +1137,14 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - FileMetaCache* file_meta_cache = nullptr, - bool enable_file_meta_memory_cache = true) const { + FileMetaCache* file_meta_cache = nullptr, bool enable_file_meta_memory_cache = true, + int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); - file_description->mtime = 123; + file_description->mtime = mtime; file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; return std::make_unique( @@ -1666,7 +1666,8 @@ TEST_F(NewParquetReaderTest, ReadMultipleRowGroups) { TEST_F(NewParquetReaderTest, RewriteSameLocalPathDoesNotReuseUnknownMtimePageCache) { RuntimeProfile first_profile("new_parquet_reader_first_unknown_mtime"); { - auto reader = create_reader(0, -1, &first_profile); + auto reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, nullptr, + true, 0); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -1693,7 +1694,8 @@ TEST_F(NewParquetReaderTest, RewriteSameLocalPathDoesNotReuseUnknownMtimePageCac // from the previous physical file, even when the query option enables parquet file page cache. write_int_pair_parquet_file(_file_path); RuntimeProfile second_profile("new_parquet_reader_second_unknown_mtime"); - auto reader = create_reader(0, -1, &second_profile); + auto reader = + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, nullptr, true, 0); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); From 48d7b109bef4c94f1d171f6a184d56734d4433ad Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 12 Jul 2026 16:51:18 +0800 Subject: [PATCH 06/11] [improvement](be) Add ORC v2 file meta disk cache ### What problem does this PR solve? Issue Number: close #xxx Related PR: #63376 Problem Summary: File meta disk cache already supports external file metadata through the FileMetaCache boundary, but format v2 ORC did not participate in the cache. This change adds an ORC_V2 file meta cache namespace, wires ordinary format v2 ORC readers through FileMetaCache, reuses ORC serialized file tail payloads for memory and persistent cache hits, and refreshes invalid persistent payloads after a successful source-file fallback. Legacy ORC keeps using the ORC namespace so both reader implementations do not share incompatible cached values. ### Release note None ### Check List (For Author) - Test: Unit Test / Manual test - ./build.sh --be -j 10 - ./run-be-ut.sh --run --filter='FileMetaCacheDiskTest.OrcFormatsUseIndependentKeys:NewOrcReaderTest.*FileMetaCache*:NewOrcReaderTest.OversizedPersistentPayloadRemainsMemoryCacheEligible' -j 10 - git diff --check - Behavior changed: Yes. Format v2 ORC file metadata can use the existing external file meta disk cache when enabled. - Does this need documentation: No --- be/src/format_v2/orc/orc_reader.cpp | 125 ++++++++- be/src/format_v2/orc/orc_reader.h | 20 +- be/src/format_v2/table_reader.cpp | 3 +- be/src/io/fs/file_meta_cache.h | 1 + be/src/io/fs/file_meta_disk_cache.cpp | 3 + .../file_reader/file_meta_cache_test.cpp | 36 +++ be/test/format_v2/orc/orc_reader_test.cpp | 258 +++++++++++++++++- 7 files changed, 439 insertions(+), 7 deletions(-) diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 32b8aed2a888db..495a163bac8a18 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -69,6 +69,7 @@ #include "exprs/vslot_ref.h" #include "format_v2/column_mapper.h" #include "format_v2/orc/orc_search_argument.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" @@ -743,10 +744,13 @@ OrcReader::OrcReader(std::shared_ptr& system_propertie std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(std::move(global_rowid_context)), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} OrcReader::~OrcReader() = default; @@ -799,6 +803,42 @@ void OrcReader::_init_profile() { ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + _orc_profile.file_footer_read_calls = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); +} + +void OrcReader::_update_file_meta_cache_profile() const { + if (_profile == nullptr) { + return; + } + COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); } void OrcReader::_collect_profile() const { @@ -866,9 +906,90 @@ Status OrcReader::init(RuntimeState* state) { _state->timezone_obj = state->timezone_obj(); } + RETURN_IF_ERROR(_init_orc_reader()); + _update_file_meta_cache_profile(); + return Status::OK(); +} + +Status OrcReader::_init_orc_reader() { + if (_file_meta_cache == nullptr) { + ++_reader_statistics.file_footer_read_calls; + return _create_orc_reader(nullptr); + } + + const std::string file_meta_cache_key = + FileMetaCache::get_key(_tracing_file_reader, *_file_description); + const int64_t file_size = _file_description->file_size == -1 + ? static_cast(_tracing_file_reader->size()) + : _file_description->file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC_V2, + .key = file_meta_cache_key, + .modification_time = _file_description->mtime, + .file_size = file_size, + .enable_memory_cache = _enable_file_meta_memory_cache && _file_meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_file_tail = meta_cache_handle.data(); + DORIS_CHECK(cached_file_tail != nullptr); + RETURN_IF_ERROR(_create_orc_reader(cached_file_tail)); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto cached_file_tail = std::make_unique(std::move(serialized_file_tail)); + Status status = _create_orc_reader(cached_file_tail.get()); + if (!status.ok()) { + if (status.is()) { + return status; + } + VLOG_DEBUG << "ignore invalid ORC v2 file meta persistent cache payload: " << status; + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr)); + cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, &meta_cache_handle, + *cached_file_tail, &file_meta_cache_profile); + } else if (file_meta_cache_context.enable_memory_cache) { + _file_meta_cache->insert(memory_cache_key, cached_file_tail, &meta_cache_handle); + } + } else { + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr)); + const uint64_t file_tail_size = + _state->reader->getFileFooterLength() + _state->reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, &meta_cache_handle, + *cached_file_tail, &file_meta_cache_profile); + } + } + return Status::OK(); +} + +Status OrcReader::_create_orc_reader(const std::string* serialized_file_tail) { + _state->reader.reset(); + _state->root_type = nullptr; ::orc::ReaderOptions options; options.setMemoryPool(*ExecEnv::GetInstance()->orc_memory_pool()); options.setReaderMetrics(&_state->reader_metrics); + if (serialized_file_tail != nullptr) { + options.setSerializedFileTail(*serialized_file_tail); + } auto input_stream = std::make_unique(_file_description->path, _tracing_file_reader, _io_ctx.get()); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index e7de8cf5c1a6fc..1a66b7c63351b1 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -33,6 +33,10 @@ namespace cctz { class time_zone; } // namespace cctz +namespace doris { +class FileMetaCache; +} + namespace doris::format::orc { struct OrcReaderScanState; @@ -45,7 +49,8 @@ class OrcReader final : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~OrcReader() override; static format::ColumnDefinition row_position_column_definition(); @@ -88,12 +93,23 @@ class OrcReader final : public format::FileReader { RuntimeProfile::Counter* read_row_groups = nullptr; // RowGroupsReadNum RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* open_file_num = nullptr; + RuntimeProfile::Counter* file_footer_read_calls = nullptr; + RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class OrcFilterImpl; void _init_profile() override; void _collect_profile() const; + void _update_file_meta_cache_profile() const; + Status _init_orc_reader(); + Status _create_orc_reader(const std::string* serialized_file_tail); DataTypePtr _convert_to_doris_type(const ::orc::Type& type) const; DataTypePtr _convert_list_to_doris_type(const ::orc::Type& type) const; @@ -175,6 +191,8 @@ class OrcReader final : public format::FileReader { OrcProfile _orc_profile; // RuntimeProfile counters std::optional _global_rowid_context; bool _enable_mapping_timestamp_tz = false; + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::orc diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 778ce39a6409f7..9c8d7211381c48 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -686,7 +686,8 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { if (_format == FileFormat::ORC) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::CSV) { diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 3092d61479092e..8c57e8cd46bab3 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -33,6 +33,7 @@ enum class FileMetaCacheFormat : uint8_t { PARQUET = 1, ORC = 2, PARQUET_V2 = 3, + ORC_V2 = 4, }; struct FileMetaCacheContext { diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp index ba7e1a5b4fcae1..a4cce91879b2f9 100644 --- a/be/src/io/fs/file_meta_disk_cache.cpp +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -48,6 +48,8 @@ std::string_view format_name(FileMetaCacheFormat format) { return "orc"; case FileMetaCacheFormat::PARQUET_V2: return "parquet_v2"; + case FileMetaCacheFormat::ORC_V2: + return "orc_v2"; } DCHECK(false) << "unknown file meta cache format"; return "unknown"; @@ -58,6 +60,7 @@ Status parse_format(uint8_t format, FileMetaCacheFormat* parsed) { case FileMetaCacheFormat::PARQUET: case FileMetaCacheFormat::ORC: case FileMetaCacheFormat::PARQUET_V2: + case FileMetaCacheFormat::ORC_V2: *parsed = static_cast(format); return Status::OK(); } diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 17cb7686ea98ff..82c4dcc2dc5b0e 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -95,6 +95,9 @@ std::string file_meta_disk_cache_key(FileMetaCacheFormat format, const std::stri case FileMetaCacheFormat::PARQUET_V2: key.append("parquet_v2"); break; + case FileMetaCacheFormat::ORC_V2: + key.append("orc_v2"); + break; } key.push_back(':'); key.append(meta_key); @@ -300,6 +303,39 @@ TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { FileMetaCacheLookupState::MEMORY_HIT); } +TEST(FileMetaCacheDiskTest, OrcFormatsUseIndependentKeys) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_orc_format_isolation_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.orc", 123, 456); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC, meta_key, 123, 456, + "legacy orc serialized tail") + .ok()); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, + "format v2 orc serialized tail") + .ok()); + + std::string legacy_payload; + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::ORC, meta_key, 123, 456, &legacy_payload).ok()); + EXPECT_EQ(legacy_payload, "legacy orc serialized tail"); + + std::string v2_payload; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, &v2_payload).ok()); + EXPECT_EQ(v2_payload, "format v2 orc serialized tail"); +} + TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 0e36498c41d9bf..33c6499117ebf2 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_array.h" @@ -69,11 +71,13 @@ #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -4235,6 +4239,67 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t return projection; } +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++_read_count; + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++_write_count; + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int read_count() const { return _read_count; } + int write_count() const { return _write_count; } + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + return std::to_string(static_cast(format)) + ":" + key; + } + + std::map _entries; + int _read_count = 0; + int _write_count = 0; + int _remove_count = 0; +}; + class NewOrcReaderTest : public testing::Test { protected: void SetUp() override { @@ -4249,14 +4314,18 @@ class NewOrcReaderTest : public testing::Test { std::unique_ptr create_reader( RuntimeProfile* profile = nullptr, - std::optional global_rowid_context = std::nullopt) const { + std::optional global_rowid_context = std::nullopt, + FileMetaCache* file_meta_cache = nullptr, bool enable_file_meta_memory_cache = true, + int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); - return std::make_unique(system_properties, file_description, - nullptr, profile, global_rowid_context); + file_description->mtime = mtime; + return std::make_unique( + system_properties, file_description, nullptr, profile, global_rowid_context, false, + file_meta_cache, enable_file_meta_memory_cache); } std::unique_ptr create_reader_for_path( @@ -4295,6 +4364,189 @@ class NewOrcReaderTest : public testing::Test { std::string _file_path; }; +TEST_F(NewOrcReaderTest, UsesFileMetaCacheForSerializedTail) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); + + RuntimeProfile second_profile("new_orc_reader_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_persistent_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_miss_disk_cache, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_write_disk_cache, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_FALSE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC, file_meta_cache_key)); + + RuntimeProfile second_profile("new_orc_reader_persistent_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::ORC_V2, file_meta_cache_key, 123, file_size, + "not a serialized orc file tail") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + EXPECT_EQ(reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_write_disk_cache, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::ORC_V2, file_meta_cache_key), + "not a serialized orc file tail"); +} + +TEST_F(NewOrcReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + +TEST_F(NewOrcReaderTest, UnstableMtimeSkipsPersistentFileMetaCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(first_reader->init(&state).ok()); + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(second_reader->init(&state).ok()); + + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->read_count(), 0); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + +TEST_F(NewOrcReaderTest, OversizedPersistentPayloadRemainsMemoryCacheEligible) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(1024 * 1024, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; From 677f3a27fe0c5c596d8c139ab87171fb117e8b11 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 12 Jul 2026 19:15:51 +0800 Subject: [PATCH 07/11] [fix](be) Harden file metadata disk cache lifecycle ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: Runtime activation could enable persistent metadata lookups after startup without constructing the required cache objects. Nested Iceberg and transactional Hive delete readers also ignored the parent memory-cache policy, and a capacity rejection during a multi-block write could leave downloaded payload blocks behind. Separate startup configuration from the runtime size gate, propagate the reader policy, preclaim complete entries before writing, and cover real disk reinitialization. ### Release note None ### Check List (For Author) - Test: Unit Tests added; modified production and test objects compiled with ASAN and -Werror; full BE UT execution pending the local rebuild - Behavior changed: Yes, runtime cache-size activation is supported and failed writes no longer leave partial entries - Does this need documentation: No --- be/src/format/table/iceberg_reader.cpp | 2 + be/src/format/table/iceberg_reader_mixin.h | 1 + .../table/transactional_hive_reader.cpp | 1 + be/src/io/fs/file_meta_cache.cpp | 6 +- be/src/io/fs/file_meta_cache.h | 1 + be/src/io/fs/file_meta_disk_cache.cpp | 16 ++- be/src/runtime/exec_env_init.cpp | 4 +- .../file_reader/file_meta_cache_test.cpp | 115 ++++++++++++++++++ .../table/iceberg/iceberg_reader_test.cpp | 44 +++++++ 9 files changed, 182 insertions(+), 8 deletions(-) diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 110a10f74c0c77..7b5e59a696821e 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -369,6 +369,7 @@ Status IcebergParquetReader::_read_position_delete_file(const TFileRangeDesc* de ParquetReader parquet_delete_reader(get_profile(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, &get_state()->timezone_obj(), get_io_ctx(), get_state(), _meta_cache); + parquet_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); // The delete file range has size=-1 (read whole file). We must disable // row group filtering before init; otherwise _do_init_reader returns EndOfFile // when _filter_groups && _range_size < 0. @@ -640,6 +641,7 @@ Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete OrcReader orc_delete_reader(get_profile(), get_state(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, get_state()->timezone(), get_io_ctx(), _meta_cache); + orc_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); OrcInitContext delete_ctx; delete_ctx.column_names = delete_file_col_names; delete_ctx.col_name_to_block_idx = diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index 2bc7be1741f756..3dc02db41eb9ff 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -480,6 +480,7 @@ Status IcebergReaderMixin::_equality_delete_base( } std::unique_ptr delete_reader = _create_equality_reader(delete_desc); + delete_reader->set_enable_file_meta_memory_cache(this->_enable_file_meta_memory_cache); RETURN_IF_ERROR(delete_reader->init_schema_reader()); std::vector equality_delete_col_names; diff --git a/be/src/format/table/transactional_hive_reader.cpp b/be/src/format/table/transactional_hive_reader.cpp index 8e31ff9b37dfdb..5282e70ac11f44 100644 --- a/be/src/format/table/transactional_hive_reader.cpp +++ b/be/src/format/table/transactional_hive_reader.cpp @@ -206,6 +206,7 @@ Status TransactionalHiveReader::on_after_init_reader(ReaderInitContext* /*ctx*/) OrcReader delete_reader(get_profile(), get_state(), get_scan_params(), delete_range, 256 /*batch_size*/, get_state()->timezone(), get_io_ctx(), _meta_cache, false); + delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); auto acid_info_node = std::make_shared(); for (auto idx = 0; idx < TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.size(); diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index a9c65dfce3f356..9d2d1b16581828 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -68,8 +68,12 @@ std::string FileMetaCache::get_memory_cache_key(FileMetaCacheFormat format, std: return memory_cache_key; } +bool FileMetaCache::is_persistent_cache_configured() { + return config::enable_external_file_meta_disk_cache; +} + bool FileMetaCache::is_persistent_cache_enabled() { - return config::enable_external_file_meta_disk_cache && + return is_persistent_cache_configured() && config::external_file_meta_disk_cache_max_entry_bytes > 0; } diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 8c57e8cd46bab3..a7c93496dde442 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -106,6 +106,7 @@ class FileMetaCache { return get_memory_cache_key(context.format, context.key); } + static bool is_persistent_cache_configured(); static bool is_persistent_cache_enabled(); static bool is_persistent_cache_payload_size_allowed(uint64_t payload_size); diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp index a4cce91879b2f9..7f196771cf0eaf 100644 --- a/be/src/io/fs/file_meta_disk_cache.cpp +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -261,21 +261,27 @@ Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& f io::CacheContext context = build_meta_cache_context(); context.stats = &stats; auto holder = cache->get_or_set(hash, 0, value.size(), context); - auto write_block = [&](const io::FileBlockSPtr& block) -> Status { + + // Claim the complete entry before appending so a rejected block cannot leave partial data. + io::FileBlocks blocks_to_write; + for (const auto& block : holder.file_blocks) { auto state = block->state(); if (state == io::FileBlock::State::DOWNLOADING && !block->is_downloader()) { state = block->wait(); } if (state == io::FileBlock::State::DOWNLOADED) { - return Status::OK(); + continue; } if (state != io::FileBlock::State::EMPTY) { return Status::NotFound("file meta disk cache block is not writable"); } - if (block->get_or_set_downloader() != io::FileBlock::get_caller_id()) { return Status::NotFound("file meta disk cache block has another downloader"); } + blocks_to_write.emplace_back(block); + } + + auto write_block = [&](const io::FileBlockSPtr& block) -> Status { const auto& range = block->range(); DCHECK_LT(range.right, value.size()); Status status = block->append(Slice(value.data() + range.left, range.size())); @@ -291,12 +297,12 @@ Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& f return Status::OK(); }; - for (const auto& block : holder.file_blocks) { + for (const auto& block : blocks_to_write) { if (block->range().left >= FILE_META_DISK_CACHE_HEADER_SIZE) { RETURN_IF_ERROR(write_block(block)); } } - for (const auto& block : holder.file_blocks) { + for (const auto& block : blocks_to_write) { if (block->range().left < FILE_META_DISK_CACHE_HEADER_SIZE) { RETURN_IF_ERROR(write_block(block)); } diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 8b2b49a195bbc6..a5121d29975c9b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -507,7 +507,7 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths "= true"; exit(-1); } - if (!FileMetaCache::is_persistent_cache_enabled()) { + if (!FileMetaCache::is_persistent_cache_configured()) { return; } } @@ -680,7 +680,7 @@ Status ExecEnv::init_mem_env() { config::file_cache_max_file_reader_cache_size = block_file_cache_fd_cache_size; _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num, - FileMetaCache::is_persistent_cache_enabled() + FileMetaCache::is_persistent_cache_configured() ? std::make_unique() : nullptr); diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 82c4dcc2dc5b0e..cf6d26dcc800a4 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -18,8 +18,12 @@ #include "io/fs/file_meta_cache.h" #include +#include +#include +#include #include +#include #include "common/config.h" #include "gmock/gmock.h" @@ -130,6 +134,13 @@ io::CacheContext create_file_meta_disk_cache_context(io::ReadStatistics* stats) context.stats = stats; return context; } + +bool wait_until_file_cache_ready(io::BlockFileCache* cache) { + for (int i = 0; i < 1000 && !cache->get_async_open_success(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return cache->get_async_open_success(); +} } // anonymous namespace TEST(FileMetaCacheTest, KeyGenerationFromParams) { @@ -303,6 +314,31 @@ TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { FileMetaCacheLookupState::MEMORY_HIT); } +TEST(FileMetaCacheTest, PersistentCacheConfigurationIsIndependentFromRuntimeSizeGate) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 0; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); + + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_TRUE(FileMetaCache::is_persistent_cache_enabled()); + + config::enable_external_file_meta_disk_cache = false; + EXPECT_FALSE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); +} + TEST(FileMetaCacheDiskTest, OrcFormatsUseIndependentKeys) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; @@ -451,6 +487,85 @@ TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) EXPECT_TRUE(output.empty()); } +TEST(FileMetaCacheDiskTest, CapacityFailureDoesNotLeavePartiallyDownloadedBlocks) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.capacity = 64; + settings.index_queue_size = 64; + settings.index_queue_elements = 8; + settings.max_query_cache_size = 0; + io::BlockFileCache block_cache("file_meta_disk_cache_capacity_failure_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/too-large.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, + std::string(80, 'p')); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(block_cache.get_blocks_by_key(hash).empty()); +} + +TEST(FileMetaCacheDiskTest, DiskEntrySurvivesBlockFileCacheReinitialization) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer restore_config {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + const std::filesystem::path cache_path = + std::filesystem::temp_directory_path() / + ("doris_file_meta_disk_cache_restart_" + std::to_string(getpid())); + std::error_code error; + std::filesystem::remove_all(cache_path, error); + ASSERT_TRUE(std::filesystem::create_directories(cache_path)); + Defer remove_cache_path {[&] { std::filesystem::remove_all(cache_path, error); }}; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.storage = "disk"; + const std::string meta_key = FileMetaCache::get_key("s3://bucket/restart.parquet", 123, 456); + const std::string payload = "serialized footer persisted across cache initialization"; + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE( + disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, payload).ok()); + } + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); + } +} + TEST(FileMetaCacheDiskTest, IncompleteMultiBlockEntryMissesWithoutInvalidating) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 89bf367d06b053..38152d94305476 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -845,6 +845,50 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } +TEST_F(IcebergReaderTest, position_delete_inherits_disabled_file_meta_memory_cache) { + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path("data.parquet"); + scan_range.__set_start_offset(0); + scan_range.__set_size(0); + + RuntimeProfile profile("test_profile"); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + + IcebergParquetReader iceberg_reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, + &io_ctx, &runtime_state, cache.get()); + iceberg_reader.set_enable_file_meta_memory_cache(false); + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(IcebergReaderMixin::POSITION_DELETE); + delete_file.__set_path(mixed_position_delete_file()); + ASSERT_TRUE(iceberg_reader + .TEST_position_delete_base("file:///tmp/non_matching_data.parquet", + {delete_file}) + .ok()); + + io::FileReaderSPtr delete_file_reader; + ASSERT_TRUE(io::global_local_filesystem() + ->open_file(mixed_position_delete_file(), &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET, meta_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(cache->lookup(memory_key, &handle)); +} + // Test reading real Iceberg Orc file using IcebergTableReader TEST_F(IcebergReaderTest, read_iceberg_orc_file) { // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email From 41062d6447572307d9b4226af839cd45b5dc9381 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 12 Jul 2026 19:19:19 +0800 Subject: [PATCH 08/11] [test](be) Add persistent file meta cache boundary tests ### What problem does this PR solve? Issue Number: None Related PR: #63376 Problem Summary: The file metadata persistent cache interface had no direct unit coverage. Add boundary-level tests for successful round trips, invalidation, profile counters, and read/write failure fallback without depending on BlockFileCache internals. ### Release note None ### Check List (For Author) - Test: Unit Test object compiled with ASAN and -Werror; full execution pending the local BE rebuild - Behavior changed: No - Does this need documentation: No --- .../file_reader/file_meta_cache_test.cpp | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index cf6d26dcc800a4..a2cdb6405189a6 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -74,6 +74,57 @@ class MockFileReader : public io::FileReader { bool _closed; }; +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++read_count; + if (fail_read) { + return Status::IOError("injected persistent cache read failure"); + } + if (!has_value || format != stored_format || key != stored_key || + modification_time != stored_modification_time || file_size != stored_file_size) { + return Status::NotFound("persistent cache entry not found"); + } + *payload = stored_payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++write_count; + if (fail_write) { + return Status::IOError("injected persistent cache write failure"); + } + stored_format = format; + stored_key = key; + stored_modification_time = modification_time; + stored_file_size = file_size; + stored_payload = payload; + has_value = true; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + ++remove_count; + if (has_value && format == stored_format && key == stored_key) { + has_value = false; + } + } + + bool fail_read = false; + bool fail_write = false; + bool has_value = false; + int read_count = 0; + int write_count = 0; + int remove_count = 0; + FileMetaCacheFormat stored_format = FileMetaCacheFormat::PARQUET; + std::string stored_key; + int64_t stored_modification_time = 0; + int64_t stored_file_size = 0; + std::string stored_payload; +}; + io::FileCacheSettings create_file_meta_disk_cache_test_settings() { io::FileCacheSettings settings; settings.capacity = 1024 * 1024; @@ -403,6 +454,112 @@ TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); } +TEST(FileMetaCacheTest, PersistentCacheRoundTripUsesFileMetaBoundary) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/persistent.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto parsed_value = std::make_unique("parsed footer"); + ObjLRUCache::CacheHandle handle; + int64_t hit_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + ASSERT_TRUE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 1); + + std::string serialized_meta; + const FileMetaCacheLookupResult lookup_result = + cache.lookup(context, &handle, &serialized_meta, &profile); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(serialized_meta, "serialized footer"); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_disk_cache, 1); + EXPECT_EQ(miss_disk_cache, 0); + + cache.invalidate_persistent_cache(context); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 1); +} + +TEST(FileMetaCacheTest, PersistentCacheFailureFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->fail_read = true; + persistent_cache_ptr->fail_write = true; + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/failure.orc", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::ORC, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(miss_disk_cache, 1); + + auto parsed_value = std::make_unique("parsed footer"); + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 0); +} + TEST(FileMetaCacheDiskTest, PersistentCacheRequiresStableModificationTime) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache; From 3f982d568bcffb86cd7bce539f2e730efa03a106 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Sun, 12 Jul 2026 19:27:45 +0800 Subject: [PATCH 09/11] [fix](be) Propagate file meta cache to Iceberg v2 delete readers ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: Iceberg format-v2 data readers received the shared file metadata cache and memory-cache policy, but format-v2 position and equality delete readers were constructed without either value. Their Parquet and ORC footer metadata therefore bypassed the persistent cache entirely. Pass the cache, memory policy, and timestamp mapping option through the delete-reader construction boundary and add enabled/disabled behavior coverage. ### Release note None ### Check List (For Author) - Test: Unit Test added; modified v2 production and test objects compiled with ASAN and -Werror; full execution pending the local BE rebuild - Behavior changed: Yes, Iceberg v2 delete-file metadata now uses the configured file metadata cache - Does this need documentation: No --- be/src/format_v2/table/iceberg_reader.cpp | 10 ++- .../format_v2/table/iceberg_reader_test.cpp | 67 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 1598fa783cda44..56c525ff3a4c0b 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -498,12 +498,16 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe auto system_properties = _delete_file_system_properties(scan_params); auto file_description = _delete_file_description(delete_range); std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); + const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && + scan_params.enable_mapping_timestamp_tz; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { *reader = std::make_unique( - system_properties, file_description, io_ctx, _scanner_profile); + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } else { - *reader = std::make_unique(system_properties, file_description, - io_ctx, _scanner_profile); + *reader = std::make_unique( + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } RETURN_IF_ERROR((*reader)->init(_runtime_state)); return Status::OK(); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 8a10fc06a41db3..1eef2dfa6c263f 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -62,6 +62,8 @@ #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/fs/file_meta_cache.h" +#include "io/fs/local_file_system.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" @@ -2018,6 +2020,71 @@ TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergDeleteReaderInheritsFileMetaMemoryCachePolicy) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_delete_file_meta_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache delete_file_cache(1); + + auto delete_footer_is_cached = [&](bool enable_file_meta_memory_cache) { + FileMetaCache file_meta_cache(1024); + doris::format::iceberg::IcebergTableReader reader; + EXPECT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_meta_cache = &file_meta_cache, + .enable_file_meta_memory_cache = enable_file_meta_memory_cache, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &delete_file_cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + EXPECT_TRUE(reader.prepare_split(split_options).ok()); + + io::FileReaderSPtr delete_file_reader; + EXPECT_TRUE(io::global_local_filesystem() + ->open_file(delete_file_path, &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET_V2, meta_key); + ObjLRUCache::CacheHandle handle; + const bool cached = file_meta_cache.lookup(memory_key, &handle); + EXPECT_TRUE(reader.close().ok()); + return cached; + }; + + EXPECT_TRUE(delete_footer_is_cached(true)); + EXPECT_FALSE(delete_footer_is_cached(false)); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_nullable_position_delete_file_test"; From a55bd3e26aa8825e90aab293963d422fb84b406e Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Mon, 13 Jul 2026 00:44:07 +0800 Subject: [PATCH 10/11] [test](be) Initialize file cache dependencies in restart test ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: The disk-cache restart test exercised FSFileCacheStorage without initializing ExecEnv's FDCache, so the restored read dereferenced a null cache under ASAN. Initialize and release the FDCache only when this test owns it, and shorten the TTL worker intervals so destruction does not wait six minutes. ### Release note None ### Check List (For Author) - Test: Unit Test - 20 focused BE tests passed under ASAN - Behavior changed: No - Does this need documentation: No --- .../file_reader/file_meta_cache_test.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index a2cdb6405189a6..24a77eb98fda19 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -31,8 +31,10 @@ #include "io/cache/block_file_cache.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" #include "io/fs/file_meta_disk_cache.h" #include "io/fs/file_reader.h" +#include "runtime/exec_env.h" #include "util/coding.h" #include "util/defer_op.h" @@ -681,13 +683,23 @@ TEST(FileMetaCacheDiskTest, DiskEntrySurvivesBlockFileCacheReinitialization) { config::enable_external_file_meta_disk_cache; const int64_t old_external_file_meta_disk_cache_max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; + const int64_t old_file_cache_background_ttl_gc_interval_ms = + config::file_cache_background_ttl_gc_interval_ms; + const int64_t old_file_cache_background_ttl_info_update_interval_ms = + config::file_cache_background_ttl_info_update_interval_ms; Defer restore_config {[&] { config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; config::external_file_meta_disk_cache_max_entry_bytes = old_external_file_meta_disk_cache_max_entry_bytes; + config::file_cache_background_ttl_gc_interval_ms = + old_file_cache_background_ttl_gc_interval_ms; + config::file_cache_background_ttl_info_update_interval_ms = + old_file_cache_background_ttl_info_update_interval_ms; }}; config::enable_external_file_meta_disk_cache = true; config::external_file_meta_disk_cache_max_entry_bytes = 1024; + config::file_cache_background_ttl_gc_interval_ms = 20; + config::file_cache_background_ttl_info_update_interval_ms = 20; const std::filesystem::path cache_path = std::filesystem::temp_directory_path() / @@ -697,6 +709,17 @@ TEST(FileMetaCacheDiskTest, DiskEntrySurvivesBlockFileCacheReinitialization) { ASSERT_TRUE(std::filesystem::create_directories(cache_path)); Defer remove_cache_path {[&] { std::filesystem::remove_all(cache_path, error); }}; + auto* exec_env = ExecEnv::GetInstance(); + const bool owns_fd_cache = exec_env->file_cache_open_fd_cache() == nullptr; + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(std::make_unique()); + } + Defer release_fd_cache {[&] { + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(nullptr); + } + }}; + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); settings.storage = "disk"; const std::string meta_key = FileMetaCache::get_key("s3://bucket/restart.parquet", 123, 456); From 6cfbea948cdf8055d2942e289de4272affff7b81 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Mon, 13 Jul 2026 21:38:47 +0800 Subject: [PATCH 11/11] [fix](be) Harden persistent file meta cache lookup ### What problem does this PR solve? Issue Number: None Related PR: #64961 Problem Summary: FileMetaCache context lookups could leave a reused memory-cache handle pinned after a persistent hit or miss. Persistent reads also accepted entries larger than the current runtime size limit. Reset the output handle before every context lookup, invalidate oversized persistent entries, and add regression coverage for both cases. ### Release note None ### Check List (For Author) - Test: Unit Test - ContextLookupClearsReusedMemoryHandle and OversizedPersistentReadFallsBackToMiss (ASAN) - Behavior changed: Yes. Non-memory lookup results now return an invalid handle, and oversized persistent entries are treated as cache misses. - Does this need documentation: No --- be/src/io/fs/file_meta_cache.cpp | 8 +- .../file_reader/file_meta_cache_test.cpp | 105 +++++++++++++++++- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index 9d2d1b16581828..ea5804059b91db 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -80,7 +80,7 @@ bool FileMetaCache::is_persistent_cache_enabled() { bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) { const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 && - payload_size > 0 && payload_size <= static_cast(max_entry_bytes); + payload_size != 0 && std::cmp_less_equal(payload_size, max_entry_bytes); } FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context, @@ -89,6 +89,7 @@ FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& cont FileMetaCacheProfile* profile) { DCHECK(handle != nullptr); DCHECK(serialized_meta != nullptr); + *handle = ObjLRUCache::CacheHandle(); if (context.enable_memory_cache && lookup(get_memory_cache_key(context), handle)) { serialized_meta->clear(); if (profile != nullptr) { @@ -142,6 +143,11 @@ bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context, VLOG_DEBUG << "lookup file meta persistent cache failed: " << status; return false; } + if (!is_persistent_cache_payload_size_allowed(static_cast(payload->size()))) { + payload->clear(); + _persistent_cache->remove(context.format, context.key); + return false; + } return true; } diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 24a77eb98fda19..f54a07aba02845 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -44,7 +44,7 @@ namespace { class MockFileReader : public io::FileReader { public: MockFileReader(const std::string& file_name, size_t size) - : _file_name(file_name), _size(size), _closed(false) {} + : _file_name(file_name), _size(size) {} ~MockFileReader() override = default; const io::Path& path() const override { @@ -73,7 +73,7 @@ class MockFileReader : public io::FileReader { private: std::string _file_name; size_t _size; - bool _closed; + bool _closed = false; }; class TestPersistentFileMetaCache final : public FileMetaPersistentCache { @@ -339,6 +339,67 @@ TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { EXPECT_EQ(miss_disk_cache, 0); } +TEST(FileMetaCacheTest, ContextLookupClearsReusedMemoryHandle) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(1024, std::move(persistent_cache)); + + const std::string memory_key = + FileMetaCache::get_key("s3://bucket/memory-handle.parquet", 123, 456); + const FileMetaCacheContext memory_context {.format = FileMetaCacheFormat::PARQUET, + .key = memory_key, + .modification_time = 123, + .file_size = 456}; + auto memory_value = std::make_unique("parsed memory footer"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE( + cache.insert(memory_context, memory_value, &insert_handle, "serialized memory footer") + .memory_inserted); + + ObjLRUCache::CacheHandle reused_handle; + std::string serialized_meta; + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + + const std::string persistent_key = + FileMetaCache::get_key("s3://bucket/persistent-handle.parquet", 123, 456); + const FileMetaCacheContext persistent_context {.format = FileMetaCacheFormat::PARQUET, + .key = persistent_key, + .modification_time = 123, + .file_size = 456}; + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = persistent_context.format; + persistent_cache_ptr->stored_key = persistent_context.key; + persistent_cache_ptr->stored_modification_time = persistent_context.modification_time; + persistent_cache_ptr->stored_file_size = persistent_context.file_size; + persistent_cache_ptr->stored_payload = "serialized persistent footer"; + + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_FALSE(reused_handle.valid()); + + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + persistent_cache_ptr->has_value = false; + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(reused_handle.valid()); +} + TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { FileMetaCache cache(1024 * 1024); const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.parquet", 123, 456); @@ -562,6 +623,46 @@ TEST(FileMetaCacheTest, PersistentCacheFailureFallsBackToMiss) { EXPECT_EQ(write_disk_cache, 0); } +TEST(FileMetaCacheTest, OversizedPersistentReadFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 4; + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/oversized.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = context.format; + persistent_cache_ptr->stored_key = context.key; + persistent_cache_ptr->stored_modification_time = context.modification_time; + persistent_cache_ptr->stored_file_size = context.file_size; + persistent_cache_ptr->stored_payload = "oversized payload"; + FileMetaCache cache(0, std::move(persistent_cache)); + + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); +} + TEST(FileMetaCacheDiskTest, PersistentCacheRequiresStableModificationTime) { const bool old_enable_external_file_meta_disk_cache = config::enable_external_file_meta_disk_cache;