diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9ec587bd100ec5..3b7745e3b97c0a 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1213,6 +1213,8 @@ DEFINE_Bool(enable_file_cache, "false"); // Both will use the directory "memory" on the disk instead of the real RAM. DEFINE_String(file_cache_path, "[{\"path\":\"${DORIS_HOME}/file_cache\"}]"); DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB +DEFINE_Bool(enable_external_file_meta_disk_cache, "true"); +DEFINE_mInt64(external_file_meta_disk_cache_max_entry_bytes, "67108864"); // 64MB DEFINE_Bool(clear_file_cache, "false"); DEFINE_mBool(enable_file_cache_query_limit, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 012f09a735c0cd..d8e4a4cf48fbcb 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1258,6 +1258,8 @@ DECLARE_Bool(enable_file_cache); // Both will use the directory "memory" on the disk instead of the real RAM. DECLARE_String(file_cache_path); DECLARE_Int64(file_cache_each_block_size); +DECLARE_Bool(enable_external_file_meta_disk_cache); +DECLARE_mInt64(external_file_meta_disk_cache_max_entry_bytes); DECLARE_Bool(clear_file_cache); DECLARE_mBool(enable_file_cache_query_limit); DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent); diff --git a/be/src/exec/scan/file_scanner.cpp b/be/src/exec/scan/file_scanner.cpp index 78d6421adb5054..5b9bf67eacf490 100644 --- a/be/src/exec/scan/file_scanner.cpp +++ b/be/src/exec/scan/file_scanner.cpp @@ -1346,6 +1346,12 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr parquet_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. + const bool enable_file_meta_memory_cache = + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; phmap::flat_hash_map>> slot_id_to_predicates = _local_state @@ -1371,6 +1377,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&pctx); _cur_reader = std::move(iceberg_reader); } else if (range.__isset.table_format_params && @@ -1379,6 +1386,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _kv_cache, _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&pctx); _cur_reader = std::move(paimon_reader); } else if (range.__isset.table_format_params && @@ -1387,6 +1395,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiParquetReader::create_unique( _profile, *_params, range, _state->batch_size(), &_state->timezone_obj(), _io_ctx, _state, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&pctx); _cur_reader = std::move(hudi_reader); } else if (range.table_format_params.table_format_type == "hive") { @@ -1398,6 +1407,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&pctx); _cur_reader = std::move(hive_reader); } else if (range.table_format_params.table_format_type == "tvf") { @@ -1411,6 +1421,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } else if (_is_load) { @@ -1420,6 +1431,7 @@ Status FileScanner::_init_parquet_reader(FileMetaCache* file_meta_cache_ptr, _io_ctx, _state, file_meta_cache_ptr, _state->query_options().enable_parquet_lazy_mat); } + configure_file_meta_memory_cache(parquet_reader.get()); init_status = static_cast(parquet_reader.get())->init_reader(&pctx); _cur_reader = std::move(parquet_reader); } @@ -1431,6 +1443,12 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, std::unique_ptr orc_reader) { const TFileRangeDesc& range = _current_range; Status init_status = Status::OK(); + // Row-id fetch scanners read one mapped range without a split source and disable memory cache. + const bool enable_file_meta_memory_cache = + _split_source != nullptr && _should_enable_file_meta_memory_cache(file_meta_cache_ptr); + auto configure_file_meta_memory_cache = [&](GenericReader* reader) { + reader->set_enable_file_meta_memory_cache(enable_file_meta_memory_cache); + }; // Build unified OrcInitContext (shared by all ORC reader variants) OrcInitContext octx; @@ -1449,6 +1467,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(tran_orc_reader.get()); init_status = static_cast(tran_orc_reader.get())->init_reader(&octx); _cur_reader = std::move(tran_orc_reader); @@ -1462,6 +1481,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(iceberg_reader.get()); init_status = static_cast(iceberg_reader.get())->init_reader(&octx); _cur_reader = std::move(iceberg_reader); @@ -1471,6 +1491,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto paimon_reader = PaimonOrcReader::create_unique( _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _kv_cache, _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(paimon_reader.get()); init_status = static_cast(paimon_reader.get())->init_reader(&octx); _cur_reader = std::move(paimon_reader); @@ -1480,6 +1501,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, auto hudi_reader = HudiOrcReader::create_unique(_profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr); + configure_file_meta_memory_cache(hudi_reader.get()); init_status = static_cast(hudi_reader.get())->init_reader(&octx); _cur_reader = std::move(hudi_reader); @@ -1493,6 +1515,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(hive_reader.get()); init_status = static_cast(hive_reader.get())->init_reader(&octx); _cur_reader = std::move(hive_reader); @@ -1507,6 +1530,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, [this]() -> std::shared_ptr { return _create_row_id_column_iterator(); }); + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } else if (_is_load) { @@ -1515,6 +1539,7 @@ Status FileScanner::_init_orc_reader(FileMetaCache* file_meta_cache_ptr, _profile, _state, *_params, range, _state->batch_size(), _state->timezone(), _io_ctx, file_meta_cache_ptr, _state->query_options().enable_orc_lazy_mat); } + configure_file_meta_memory_cache(orc_reader.get()); init_status = static_cast(orc_reader.get())->init_reader(&octx); _cur_reader = std::move(orc_reader); } diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index 3675fd2449711e..2fd6e00ac21e21 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -326,12 +326,16 @@ class FileScanner : public Scanner { : _local_state->get_push_down_agg_type(); } - // enable the file meta cache only when - // 1. max_external_file_meta_cache_num is > 0 - // 2. the file number is less than 1/3 of cache's capacibility - // Otherwise, the cache miss rate will be high bool _should_enable_file_meta_cache() { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); + } + + // Enable memory file meta cache only when the file number is less than 1/3 of cache capacity. + // Otherwise, the cache miss rate will be high. Persistent cache is not gated by this rule. + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 10b5f850ea36f7..953ebd2351da63 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -438,6 +438,11 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); + FileMetaCache* file_meta_cache = nullptr; + if (_should_enable_file_meta_cache()) { + file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + DORIS_CHECK(file_meta_cache != nullptr); + } RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), @@ -449,6 +454,8 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .file_slot_descs = &_file_slot_descs, .push_down_agg_type = _local_state->get_push_down_agg_type(), .condition_cache_digest = _local_state->get_condition_cache_digest(), + .file_meta_cache = file_meta_cache, + .enable_file_meta_memory_cache = _should_enable_file_meta_memory_cache(file_meta_cache), })); return Status::OK(); } @@ -512,7 +519,13 @@ bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not } bool FileScannerV2::_should_enable_file_meta_cache() const { - return ExecEnv::GetInstance()->file_meta_cache()->enabled() && + auto* file_meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + return file_meta_cache != nullptr && + (file_meta_cache->enabled() || FileMetaCache::is_persistent_cache_enabled()); +} + +bool FileScannerV2::_should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const { + return file_meta_cache != nullptr && file_meta_cache->enabled() && _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 66242b610f4373..d4f067bae3910b 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -120,6 +120,7 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); bool _should_enable_file_meta_cache() const; + bool _should_enable_file_meta_memory_cache(FileMetaCache* file_meta_cache) const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; Status _generate_partition_values(const TFileRangeDesc& range, diff --git a/be/src/format/generic_reader.h b/be/src/format/generic_reader.h index cc21ce82c2805e..2f0589c4e6a235 100644 --- a/be/src/format/generic_reader.h +++ b/be/src/format/generic_reader.h @@ -258,6 +258,8 @@ class GenericReader : public ProfileCollector { // Pass condition cache context to the reader for HIT/MISS tracking. virtual void set_condition_cache_context(std::shared_ptr ctx) {} + void set_enable_file_meta_memory_cache(bool enable) { _enable_file_meta_memory_cache = enable; } + // Returns true if this reader can produce an accurate total row count from metadata // without reading actual data. Used to determine if CountReader decorator can be applied. // Only ORC and Parquet readers support this (via file footer metadata). @@ -279,6 +281,7 @@ class GenericReader : public ProfileCollector { // Cache to save some common part such as file footer. // Maybe null if not used FileMetaCache* _meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; // ---- Column descriptors (set by init_reader, owned by FileScanner) ---- const std::vector* _column_descs = nullptr; diff --git a/be/src/format/orc/vorc_reader.cpp b/be/src/format/orc/vorc_reader.cpp index 28ef930057f3a2..09dfd720476efb 100644 --- a/be/src/format/orc/vorc_reader.cpp +++ b/be/src/format/orc/vorc_reader.cpp @@ -336,6 +336,18 @@ void OrcReader::_collect_profile_before_close() { COUNTER_UPDATE(_orc_profile.lazy_read_filtered_rows, _statistics.lazy_read_filtered_rows); COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _statistics.file_footer_read_calls); COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _statistics.file_footer_write_disk_cache_time); if (_file_input_stream != nullptr) { _file_input_stream->collect_profile_before_close(); } @@ -376,6 +388,18 @@ void OrcReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _orc_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); } } @@ -439,21 +463,62 @@ Status OrcReader::_create_file_reader() { RETURN_IF_ERROR(create_orc_reader()); } else { auto inner_file_reader = _file_input_stream->get_inner_reader(); - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(inner_file_reader, _file_description); + const int64_t file_size = _file_description.file_size == -1 ? inner_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = _enable_file_meta_memory_cache && _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_statistics.file_footer_hit_cache, + .hit_memory_cache = &_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); // Local variables can be required because setSerializedFileTail is an assignment operation, not a reference. ObjLRUCache::CacheHandle _meta_cache_handle; - if (_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { - const std::string* footer_ptr = _meta_cache_handle.data(); + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const std::string* footer_ptr = _meta_cache_handle.data(); options.setSerializedFileTail(*footer_ptr); RETURN_IF_ERROR(create_orc_reader()); - _statistics.file_footer_hit_cache++; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto footer_ptr = std::make_unique(std::move(serialized_file_tail)); + options.setSerializedFileTail(*footer_ptr); + Status status = create_orc_reader(); + if (!status.ok()) { + VLOG_DEBUG << "ignore invalid orc file meta persistent cache payload: " << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _reader.reset(); + _file_input_stream.reset(); + return _create_file_reader(); + } + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(memory_cache_key, footer_ptr, &_meta_cache_handle); + } } else { _statistics.file_footer_read_calls++; RETURN_IF_ERROR(create_orc_reader()); - std::string* footer_ptr = new std::string {_reader->getSerializedFileTail()}; - _meta_cache->insert(file_meta_cache_key, footer_ptr, &_meta_cache_handle); + const uint64_t file_tail_size = + _reader->getFileFooterLength() + _reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto footer_ptr = std::make_unique(_reader->getSerializedFileTail()); + _meta_cache->insert(file_meta_cache_context, footer_ptr, &_meta_cache_handle, + *footer_ptr, &file_meta_cache_profile); + } } } diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h index 6d64f604cdee5a..e8c006e33dc2fe 100644 --- a/be/src/format/orc/vorc_reader.h +++ b/be/src/format/orc/vorc_reader.h @@ -150,6 +150,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { int64_t lazy_read_filtered_rows = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; }; OrcReader(RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, @@ -317,6 +323,12 @@ class OrcReader : public TableFormatReader, public RowPositionProvider { RuntimeProfile::Counter* evaluated_row_group_count = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class ORCFilterImpl : public orc::ORCFilter { diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 26caf7faac66f2..09ca47da5853c4 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -65,6 +65,7 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "util/slice.h" #include "util/string_util.h" +#include "util/thrift_util.h" #include "util/timezone_utils.h" namespace cctz { @@ -255,6 +256,18 @@ void ParquetReader::_init_profile() { ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); _parquet_profile.file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _parquet_profile.file_footer_read_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + _parquet_profile.file_footer_write_disk_cache_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); _parquet_profile.decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime", parquet_profile, 1); _parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -367,21 +380,90 @@ Status ParquetReader::_open_file() { // parse magic number & parse meta data _reader_statistics.file_footer_read_calls += 1; } else { - const auto& file_meta_cache_key = + const std::string file_meta_cache_key = FileMetaCache::get_key(_tracing_file_reader, _file_description); - if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { + const int64_t file_size = _file_description.file_size == -1 + ? _tracing_file_reader->size() + : _file_description.file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET, + .key = file_meta_cache_key, + .modification_time = _file_description.mtime, + .file_size = file_size, + .enable_memory_cache = + _enable_file_meta_memory_cache && _meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); + std::string footer_payload; + const FileMetaCacheLookupResult lookup_result = + _meta_cache->lookup(file_meta_cache_context, &_meta_cache_handle, + &footer_payload, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + _file_metadata = _meta_cache_handle.data(); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + uint32_t metadata_size = static_cast(footer_payload.size()); + tparquet::FileMetaData t_metadata; + Status status = deserialize_thrift_msg( + reinterpret_cast(footer_payload.data()), &metadata_size, + true, &t_metadata); + if (status.ok()) { + _file_metadata_ptr = std::make_unique(t_metadata, metadata_size); + status = _file_metadata_ptr->init_schema(enable_mapping_varbinary, + enable_mapping_timestamp_tz); + } + if (status.ok()) { + if (file_meta_cache_context.enable_memory_cache) { + _meta_cache->insert(memory_cache_key, _file_metadata_ptr.release(), + &_meta_cache_handle); + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else { + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << status; + _meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _file_metadata_ptr.reset(); + footer_payload.clear(); + } + } + if (_file_metadata == nullptr) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, enable_mapping_timestamp_tz)); - // _file_metadata_ptr.release() : move control of _file_metadata to _meta_cache_handle - _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), - &_meta_cache_handle); - _file_metadata = _meta_cache_handle.data(); + const size_t serialized_meta_size = meta_size >= PARQUET_FOOTER_SIZE + ? meta_size - PARQUET_FOOTER_SIZE + : meta_size; + if (FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(serialized_meta_size))) { + tparquet::FileMetaData thrift_metadata = _file_metadata_ptr->to_thrift(); + ThriftSerializer serializer(true, static_cast(serialized_meta_size)); + RETURN_IF_ERROR(serializer.serialize(&thrift_metadata, &footer_payload)); + const auto insert_result = _meta_cache->insert( + file_meta_cache_context, _file_metadata_ptr, &_meta_cache_handle, + footer_payload, &file_meta_cache_profile); + if (insert_result.memory_inserted) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } + } else if (file_meta_cache_context.enable_memory_cache && + _meta_cache->insert(memory_cache_key, _file_metadata_ptr, + &_meta_cache_handle)) { + _file_metadata = _meta_cache_handle.data(); + } else { + _file_metadata = _file_metadata_ptr.get(); + } _reader_statistics.file_footer_read_calls += 1; - } else { - _reader_statistics.file_footer_hit_cache++; } - _file_metadata = _meta_cache_handle.data(); } if (_file_metadata == nullptr) { @@ -1742,6 +1824,18 @@ void ParquetReader::_collect_profile() { _reader_statistics.file_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); COUNTER_UPDATE(_parquet_profile.skip_page_header_num, _column_statistics.skip_page_header_num); COUNTER_UPDATE(_parquet_profile.parse_page_header_num, diff --git a/be/src/format/parquet/vparquet_reader.h b/be/src/format/parquet/vparquet_reader.h index 9046641c6c0c51..e6c768c2847784 100644 --- a/be/src/format/parquet/vparquet_reader.h +++ b/be/src/format/parquet/vparquet_reader.h @@ -108,6 +108,12 @@ class ParquetReader : public TableFormatReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; @@ -281,6 +287,12 @@ class ParquetReader : public TableFormatReader { RuntimeProfile::Counter* parse_page_index_time = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* decompress_time = nullptr; RuntimeProfile::Counter* decompress_cnt = nullptr; RuntimeProfile::Counter* page_read_counter = nullptr; diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 110a10f74c0c77..7b5e59a696821e 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -369,6 +369,7 @@ Status IcebergParquetReader::_read_position_delete_file(const TFileRangeDesc* de ParquetReader parquet_delete_reader(get_profile(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, &get_state()->timezone_obj(), get_io_ctx(), get_state(), _meta_cache); + parquet_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); // The delete file range has size=-1 (read whole file). We must disable // row group filtering before init; otherwise _do_init_reader returns EndOfFile // when _filter_groups && _range_size < 0. @@ -640,6 +641,7 @@ Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete OrcReader orc_delete_reader(get_profile(), get_state(), get_scan_params(), *delete_range, READ_DELETE_FILE_BATCH_SIZE, get_state()->timezone(), get_io_ctx(), _meta_cache); + orc_delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); OrcInitContext delete_ctx; delete_ctx.column_names = delete_file_col_names; delete_ctx.col_name_to_block_idx = diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index 2bc7be1741f756..3dc02db41eb9ff 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -480,6 +480,7 @@ Status IcebergReaderMixin::_equality_delete_base( } std::unique_ptr delete_reader = _create_equality_reader(delete_desc); + delete_reader->set_enable_file_meta_memory_cache(this->_enable_file_meta_memory_cache); RETURN_IF_ERROR(delete_reader->init_schema_reader()); std::vector equality_delete_col_names; diff --git a/be/src/format/table/transactional_hive_reader.cpp b/be/src/format/table/transactional_hive_reader.cpp index 8e31ff9b37dfdb..5282e70ac11f44 100644 --- a/be/src/format/table/transactional_hive_reader.cpp +++ b/be/src/format/table/transactional_hive_reader.cpp @@ -206,6 +206,7 @@ Status TransactionalHiveReader::on_after_init_reader(ReaderInitContext* /*ctx*/) OrcReader delete_reader(get_profile(), get_state(), get_scan_params(), delete_range, 256 /*batch_size*/, get_state()->timezone(), get_io_ctx(), _meta_cache, false); + delete_reader.set_enable_file_meta_memory_cache(_enable_file_meta_memory_cache); auto acid_info_node = std::make_shared(); for (auto idx = 0; idx < TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE.size(); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 135df2820145e3..522f10abbe2848 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -227,6 +227,12 @@ class FileReader { int64_t parse_footer_time = 0; int64_t file_footer_read_calls = 0; int64_t file_footer_hit_cache = 0; + int64_t file_footer_hit_memory_cache = 0; + int64_t file_footer_hit_disk_cache = 0; + int64_t file_footer_miss_disk_cache = 0; + int64_t file_footer_write_disk_cache = 0; + int64_t file_footer_read_disk_cache_time = 0; + int64_t file_footer_write_disk_cache_time = 0; int64_t file_reader_create_time = 0; int64_t open_file_num = 0; int64_t row_group_filter_time = 0; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 32b8aed2a888db..495a163bac8a18 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -69,6 +69,7 @@ #include "exprs/vslot_ref.h" #include "format_v2/column_mapper.h" #include "format_v2/orc/orc_search_argument.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" @@ -743,10 +744,13 @@ OrcReader::OrcReader(std::shared_ptr& system_propertie std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(std::move(global_rowid_context)), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} OrcReader::~OrcReader() = default; @@ -799,6 +803,42 @@ void OrcReader::_init_profile() { ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); _orc_profile.open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + _orc_profile.file_footer_read_calls = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterReadCalls", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + _orc_profile.file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(_profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + _orc_profile.file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterReadDiskCacheTime", orc_profile, 1); + _orc_profile.file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileFooterWriteDiskCacheTime", orc_profile, 1); +} + +void OrcReader::_update_file_meta_cache_profile() const { + if (_profile == nullptr) { + return; + } + COUNTER_UPDATE(_orc_profile.file_footer_read_calls, _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_orc_profile.file_footer_hit_cache, _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_orc_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_orc_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_orc_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); } void OrcReader::_collect_profile() const { @@ -866,9 +906,90 @@ Status OrcReader::init(RuntimeState* state) { _state->timezone_obj = state->timezone_obj(); } + RETURN_IF_ERROR(_init_orc_reader()); + _update_file_meta_cache_profile(); + return Status::OK(); +} + +Status OrcReader::_init_orc_reader() { + if (_file_meta_cache == nullptr) { + ++_reader_statistics.file_footer_read_calls; + return _create_orc_reader(nullptr); + } + + const std::string file_meta_cache_key = + FileMetaCache::get_key(_tracing_file_reader, *_file_description); + const int64_t file_size = _file_description->file_size == -1 + ? static_cast(_tracing_file_reader->size()) + : _file_description->file_size; + const FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::ORC_V2, + .key = file_meta_cache_key, + .modification_time = _file_description->mtime, + .file_size = file_size, + .enable_memory_cache = _enable_file_meta_memory_cache && _file_meta_cache->enabled()}; + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(file_meta_cache_context); + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_file_tail; + const FileMetaCacheLookupResult lookup_result = + _file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_file_tail, &file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_file_tail = meta_cache_handle.data(); + DORIS_CHECK(cached_file_tail != nullptr); + RETURN_IF_ERROR(_create_orc_reader(cached_file_tail)); + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + auto cached_file_tail = std::make_unique(std::move(serialized_file_tail)); + Status status = _create_orc_reader(cached_file_tail.get()); + if (!status.ok()) { + if (status.is()) { + return status; + } + VLOG_DEBUG << "ignore invalid ORC v2 file meta persistent cache payload: " << status; + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr)); + cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, &meta_cache_handle, + *cached_file_tail, &file_meta_cache_profile); + } else if (file_meta_cache_context.enable_memory_cache) { + _file_meta_cache->insert(memory_cache_key, cached_file_tail, &meta_cache_handle); + } + } else { + ++_reader_statistics.file_footer_read_calls; + RETURN_IF_ERROR(_create_orc_reader(nullptr)); + const uint64_t file_tail_size = + _state->reader->getFileFooterLength() + _state->reader->getFilePostscriptLength(); + if (file_meta_cache_context.enable_memory_cache || + FileMetaCache::is_persistent_cache_payload_size_allowed(file_tail_size)) { + auto cached_file_tail = + std::make_unique(_state->reader->getSerializedFileTail()); + _file_meta_cache->insert(file_meta_cache_context, cached_file_tail, &meta_cache_handle, + *cached_file_tail, &file_meta_cache_profile); + } + } + return Status::OK(); +} + +Status OrcReader::_create_orc_reader(const std::string* serialized_file_tail) { + _state->reader.reset(); + _state->root_type = nullptr; ::orc::ReaderOptions options; options.setMemoryPool(*ExecEnv::GetInstance()->orc_memory_pool()); options.setReaderMetrics(&_state->reader_metrics); + if (serialized_file_tail != nullptr) { + options.setSerializedFileTail(*serialized_file_tail); + } auto input_stream = std::make_unique(_file_description->path, _tracing_file_reader, _io_ctx.get()); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index e7de8cf5c1a6fc..1a66b7c63351b1 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -33,6 +33,10 @@ namespace cctz { class time_zone; } // namespace cctz +namespace doris { +class FileMetaCache; +} + namespace doris::format::orc { struct OrcReaderScanState; @@ -45,7 +49,8 @@ class OrcReader final : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~OrcReader() override; static format::ColumnDefinition row_position_column_definition(); @@ -88,12 +93,23 @@ class OrcReader final : public format::FileReader { RuntimeProfile::Counter* read_row_groups = nullptr; // RowGroupsReadNum RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* open_file_num = nullptr; + RuntimeProfile::Counter* file_footer_read_calls = nullptr; + RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; }; class OrcFilterImpl; void _init_profile() override; void _collect_profile() const; + void _update_file_meta_cache_profile() const; + Status _init_orc_reader(); + Status _create_orc_reader(const std::string* serialized_file_tail); DataTypePtr _convert_to_doris_type(const ::orc::Type& type) const; DataTypePtr _convert_list_to_doris_type(const ::orc::Type& type) const; @@ -175,6 +191,8 @@ class OrcReader final : public format::FileReader { OrcProfile _orc_profile; // RuntimeProfile counters std::optional _global_rowid_context; bool _enable_mapping_timestamp_tz = false; + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::orc diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index d80dc58181d4f6..caf8c2f6d2db5e 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -32,9 +32,11 @@ #include "common/check.h" #include "common/config.h" +#include "common/logging.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" @@ -533,19 +535,93 @@ Status arrow_status_to_doris_status(const arrow::Status& status) { } Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { + bool enable_page_cache, const io::FileDescription& file_description, + FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, + int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache) { DORIS_CHECK(input_file_reader != nullptr); + DORIS_CHECK(file_footer_read_calls != nullptr); auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); + const std::string file_meta_cache_key = + file_meta_cache != nullptr ? FileMetaCache::get_key(input_file_reader, file_description) + : std::string {}; + const int64_t file_size = file_description.file_size == -1 + ? static_cast(input_file_reader->size()) + : file_description.file_size; + std::shared_ptr<::parquet::FileMetaData> cached_metadata; + bool metadata_from_cache = false; + FileMetaCacheContext file_meta_cache_context { + .format = FileMetaCacheFormat::PARQUET_V2, + .key = file_meta_cache_key, + .modification_time = file_description.mtime, + .file_size = file_size, + .enable_memory_cache = enable_file_meta_memory_cache && file_meta_cache != nullptr && + file_meta_cache->enabled()}; + const std::string memory_cache_key = + file_meta_cache != nullptr + ? FileMetaCache::get_memory_cache_key(file_meta_cache_context) + : std::string {}; + const auto reader_properties = ::parquet::default_reader_properties(); try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); + if (file_meta_cache != nullptr) { + ObjLRUCache::CacheHandle meta_cache_handle; + std::string serialized_metadata; + const FileMetaCacheLookupResult lookup_result = + file_meta_cache->lookup(file_meta_cache_context, &meta_cache_handle, + &serialized_metadata, file_meta_cache_profile); + if (lookup_result.state == FileMetaCacheLookupState::MEMORY_HIT) { + const auto* cached_metadata_ptr = + meta_cache_handle.data>(); + DORIS_CHECK(cached_metadata_ptr != nullptr); + cached_metadata = *cached_metadata_ptr; + metadata_from_cache = true; + } else if (lookup_result.state == FileMetaCacheLookupState::PERSISTED_HIT) { + try { + uint32_t metadata_len = static_cast(serialized_metadata.size()); + cached_metadata = ::parquet::FileMetaData::Make( + serialized_metadata.data(), &metadata_len, reader_properties); + if (file_meta_cache_context.enable_memory_cache) { + auto cached_metadata_holder = + std::make_unique>( + cached_metadata); + ObjLRUCache::CacheHandle insert_handle; + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, + &insert_handle); + } + metadata_from_cache = true; + } catch (const std::exception& e) { + VLOG_DEBUG << "ignore invalid parquet file meta persistent cache payload: " + << e.what(); + file_meta_cache->invalidate_persistent_cache(file_meta_cache_context); + cached_metadata.reset(); + } + } + } + arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, + enable_page_cache, + std::move(page_cache_file_key)); + this->file_reader = + ::parquet::ParquetFileReader::Open(arrow_file, reader_properties, cached_metadata); metadata = this->file_reader->metadata(); schema = metadata != nullptr ? metadata->schema() : nullptr; + if (!metadata_from_cache) { + ++(*file_footer_read_calls); + if (file_meta_cache != nullptr && metadata != nullptr) { + auto cached_metadata_holder = + std::make_unique>(metadata); + ObjLRUCache::CacheHandle insert_handle; + if (FileMetaCache::is_persistent_cache_payload_size_allowed(metadata->size())) { + std::string serialized_metadata = metadata->SerializeToString(); + file_meta_cache->insert(file_meta_cache_context, cached_metadata_holder, + &insert_handle, serialized_metadata, + file_meta_cache_profile); + } else if (file_meta_cache_context.enable_memory_cache) { + file_meta_cache->insert(memory_cache_key, cached_metadata_holder, + &insert_handle); + } + } + } } catch (const ::parquet::ParquetException& e) { if (io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()).find("stop") != std::string_view::npos) { diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 0dca52244957d7..00e5277b8811bf 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -32,6 +32,8 @@ struct IOContext; } // namespace doris::io namespace doris { +class FileMetaCache; +struct FileMetaCacheProfile; class RuntimeProfile; } // namespace doris @@ -112,7 +114,9 @@ struct ParquetFileContext { const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); + const io::FileDescription& file_description, FileMetaCache* file_meta_cache, + FileMetaCacheProfile* file_meta_cache_profile, int64_t* file_footer_read_calls, + bool enable_file_meta_memory_cache = true); // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page // decoding, so v2 caches the serialized bytes read inside these ranges and excludes // footer/metadata reads that happen before registration. diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index d41ff295ec4b3e..3ee99db7619824 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -103,6 +103,18 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_hit_memory_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitMemoryCache", TUnit::UNIT, 1); + file_footer_hit_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitDiskCache", TUnit::UNIT, 1); + file_footer_miss_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterMissDiskCache", TUnit::UNIT, 1); + file_footer_write_disk_cache = + ADD_COUNTER_WITH_LEVEL(profile, "FileFooterWriteDiskCache", TUnit::UNIT, 1); + file_footer_read_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterReadDiskCacheTime", parquet_profile, 1); + file_footer_write_disk_cache_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileFooterWriteDiskCacheTime", parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 27f9818f4a0dcd..8e5edba1b1ba60 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -116,6 +116,12 @@ struct ParquetProfile { RuntimeProfile::Counter* open_file_num = nullptr; RuntimeProfile::Counter* file_footer_read_calls = nullptr; RuntimeProfile::Counter* file_footer_hit_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_memory_cache = nullptr; + RuntimeProfile::Counter* file_footer_hit_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_miss_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache = nullptr; + RuntimeProfile::Counter* file_footer_read_disk_cache_time = nullptr; + RuntimeProfile::Counter* file_footer_write_disk_cache_time = nullptr; RuntimeProfile::Counter* row_group_filter_time = nullptr; RuntimeProfile::Counter* page_index_read_calls = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 82ad3f78891c80..384f04c946c5da 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -40,6 +40,7 @@ #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/column_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -312,10 +313,13 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, FileMetaCache* file_meta_cache, + bool enable_file_meta_memory_cache) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _file_meta_cache(file_meta_cache), + _enable_file_meta_memory_cache(enable_file_meta_memory_cache) {} ParquetReader::~ParquetReader() = default; @@ -348,8 +352,36 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + FileMetaCacheProfile file_meta_cache_profile { + .hit_cache = &_reader_statistics.file_footer_hit_cache, + .hit_memory_cache = &_reader_statistics.file_footer_hit_memory_cache, + .hit_disk_cache = &_reader_statistics.file_footer_hit_disk_cache, + .miss_disk_cache = &_reader_statistics.file_footer_miss_disk_cache, + .write_disk_cache = &_reader_statistics.file_footer_write_disk_cache, + .read_disk_cache_time = &_reader_statistics.file_footer_read_disk_cache_time, + .write_disk_cache_time = &_reader_statistics.file_footer_write_disk_cache_time}; + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _file_meta_cache, &file_meta_cache_profile, &_reader_statistics.file_footer_read_calls, + _enable_file_meta_memory_cache)); + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _reader_statistics.file_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _reader_statistics.file_footer_hit_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_memory_cache, + _reader_statistics.file_footer_hit_memory_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_disk_cache, + _reader_statistics.file_footer_hit_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_miss_disk_cache, + _reader_statistics.file_footer_miss_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache, + _reader_statistics.file_footer_write_disk_cache); + COUNTER_UPDATE(_parquet_profile.file_footer_read_disk_cache_time, + _reader_statistics.file_footer_read_disk_cache_time); + COUNTER_UPDATE(_parquet_profile.file_footer_write_disk_cache_time, + _reader_statistics.file_footer_write_disk_cache_time); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier RETURN_IF_ERROR( diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..654b8850cd26f3 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -27,6 +27,7 @@ #include "format_v2/parquet/parquet_scan.h" namespace doris { +class FileMetaCache; namespace io { struct IOContext; } // namespace io @@ -46,7 +47,9 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, + FileMetaCache* file_meta_cache = nullptr, + bool enable_file_meta_memory_cache = true); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +92,8 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 3600b0ff706f08..961757c16a5776 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -126,6 +126,8 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, }); } diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 1598fa783cda44..56c525ff3a4c0b 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -498,12 +498,16 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe auto system_properties = _delete_file_system_properties(scan_params); auto file_description = _delete_file_description(delete_range); std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); + const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && + scan_params.enable_mapping_timestamp_tz; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { *reader = std::make_unique( - system_properties, file_description, io_ctx, _scanner_profile); + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } else { - *reader = std::make_unique(system_properties, file_description, - io_ctx, _scanner_profile); + *reader = std::make_unique( + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz, _file_meta_cache, _enable_file_meta_memory_cache); } RETURN_IF_ERROR((*reader)->init(_runtime_state)); return Status::OK(); diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index c2114968946e1b..9c0ca96ec70a12 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -154,6 +154,8 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, .scanner_profile = _scanner_profile, .push_down_agg_type = _push_down_agg_type, .condition_cache_digest = _condition_cache_digest, + .file_meta_cache = _file_meta_cache, + .enable_file_meta_memory_cache = _enable_file_meta_memory_cache, }); } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 11e2b30df6de23..9c8d7211381c48 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -470,6 +470,8 @@ Status TableReader::init(TableReadOptions&& options) { _io_ctx = options.io_ctx; _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; + _file_meta_cache = options.file_meta_cache; + _enable_file_meta_memory_cache = options.enable_file_meta_memory_cache; _file_slot_descs = options.file_slot_descs; _push_down_agg_type = options.push_down_agg_type; _condition_cache_digest = options.condition_cache_digest; @@ -677,13 +679,15 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { if (_format == FileFormat::PARQUET) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::ORC) { *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, _file_meta_cache, + _enable_file_meta_memory_cache); return Status::OK(); } if (_format == FileFormat::CSV) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index da2cb351ff68ce..cb6416e1c60ce9 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -69,6 +69,7 @@ namespace doris { class Block; +class FileMetaCache; struct DeleteFileDesc; class RuntimeState; } // namespace doris @@ -137,6 +138,8 @@ struct TableReadOptions { const TPushAggOp::type push_down_agg_type = TPushAggOp::type::NONE; // Digest of stable pushed-down predicates. A zero digest disables condition cache. uint64_t condition_cache_digest = 0; + FileMetaCache* file_meta_cache = nullptr; + bool enable_file_meta_memory_cache = true; }; struct SplitReadOptions { @@ -1525,6 +1528,8 @@ class TableReader { std::shared_ptr _io_ctx; RuntimeState* _runtime_state; RuntimeProfile* _scanner_profile; + FileMetaCache* _file_meta_cache = nullptr; + bool _enable_file_meta_memory_cache = true; const std::vector* _file_slot_descs = nullptr; FileFormat _format; TPushAggOp::type _push_down_agg_type = TPushAggOp::type::NONE; diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index f97b2f80cd6ee6..ea5804059b91db 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -17,19 +17,39 @@ #include "io/fs/file_meta_cache.h" +#include + +#include "common/config.h" +#include "common/logging.h" +#include "util/stopwatch.hpp" + namespace doris { +namespace { + +void update_profile_counter(int64_t* counter, int64_t value = 1) { + if (counter != nullptr) { + *counter += value; + } +} + +bool is_persistent_cache_identity_stable(const FileMetaCacheContext& context) { + return context.modification_time > 0; +} + +} // namespace + +FileMetaCache::FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache) + : _cache(capacity), _persistent_cache(std::move(persistent_cache)) {} std::string FileMetaCache::get_key(const std::string file_name, int64_t modification_time, int64_t file_size) { std::string meta_cache_key; - meta_cache_key.resize(file_name.size() + sizeof(int64_t)); + meta_cache_key.resize(file_name.size() + 2 * sizeof(int64_t)); memcpy(meta_cache_key.data(), file_name.data(), file_name.size()); - if (modification_time != 0) { - memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); - } else { - memcpy(meta_cache_key.data() + file_name.size(), &file_size, sizeof(int64_t)); - } + memcpy(meta_cache_key.data() + file_name.size(), &modification_time, sizeof(int64_t)); + memcpy(meta_cache_key.data() + file_name.size() + sizeof(int64_t), &file_size, sizeof(int64_t)); return meta_cache_key; } @@ -40,4 +60,116 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } +std::string FileMetaCache::get_memory_cache_key(FileMetaCacheFormat format, std::string_view key) { + std::string memory_cache_key; + memory_cache_key.reserve(key.size() + sizeof(uint8_t)); + memory_cache_key.push_back(static_cast(format)); + memory_cache_key.append(key.data(), key.size()); + return memory_cache_key; +} + +bool FileMetaCache::is_persistent_cache_configured() { + return config::enable_external_file_meta_disk_cache; +} + +bool FileMetaCache::is_persistent_cache_enabled() { + return is_persistent_cache_configured() && + config::external_file_meta_disk_cache_max_entry_bytes > 0; +} + +bool FileMetaCache::is_persistent_cache_payload_size_allowed(uint64_t payload_size) { + const int64_t max_entry_bytes = config::external_file_meta_disk_cache_max_entry_bytes; + return config::enable_external_file_meta_disk_cache && max_entry_bytes > 0 && + payload_size != 0 && std::cmp_less_equal(payload_size, max_entry_bytes); +} + +FileMetaCacheLookupResult FileMetaCache::lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, + std::string* serialized_meta, + FileMetaCacheProfile* profile) { + DCHECK(handle != nullptr); + DCHECK(serialized_meta != nullptr); + *handle = ObjLRUCache::CacheHandle(); + if (context.enable_memory_cache && lookup(get_memory_cache_key(context), handle)) { + serialized_meta->clear(); + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_memory_cache); + } + return {.state = FileMetaCacheLookupState::MEMORY_HIT}; + } + + FileMetaCacheLookupResult result; + int64_t persisted_read_time = 0; + if (lookup_persistent_cache(context, serialized_meta, &persisted_read_time)) { + result.state = FileMetaCacheLookupState::PERSISTED_HIT; + if (profile != nullptr) { + update_profile_counter(profile->hit_cache); + update_profile_counter(profile->hit_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); + } + } else if (is_persistent_cache_enabled() && is_persistent_cache_identity_stable(context) && + profile != nullptr) { + update_profile_counter(profile->miss_disk_cache); + update_profile_counter(profile->read_disk_cache_time, persisted_read_time); + } + return result; +} + +void FileMetaCache::invalidate_persistent_cache(const FileMetaCacheContext& context) { + if (_persistent_cache != nullptr) { + _persistent_cache->remove(context.format, context.key); + } +} + +bool FileMetaCache::lookup_persistent_cache(const FileMetaCacheContext& context, + std::string* payload, int64_t* read_time) { + DCHECK(payload != nullptr); + DCHECK(read_time != nullptr); + payload->clear(); + *read_time = 0; + if (!is_persistent_cache_enabled() || _persistent_cache == nullptr || + !is_persistent_cache_identity_stable(context)) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->read(context.format, context.key, context.modification_time, + context.file_size, payload); + *read_time = watch.elapsed_time(); + if (!status.ok()) { + payload->clear(); + VLOG_DEBUG << "lookup file meta persistent cache failed: " << status; + return false; + } + if (!is_persistent_cache_payload_size_allowed(static_cast(payload->size()))) { + payload->clear(); + _persistent_cache->remove(context.format, context.key); + return false; + } + return true; +} + +bool FileMetaCache::insert_persistent_cache(const FileMetaCacheContext& context, + std::string_view payload, int64_t* write_time) { + DCHECK(write_time != nullptr); + *write_time = 0; + if (!is_persistent_cache_payload_size_allowed(static_cast(payload.size())) || + _persistent_cache == nullptr || !is_persistent_cache_identity_stable(context)) { + return false; + } + + MonotonicStopWatch watch; + watch.start(); + Status status = _persistent_cache->write(context.format, context.key, context.modification_time, + context.file_size, payload); + *write_time = watch.elapsed_time(); + if (!status.ok()) { + VLOG_DEBUG << "insert file meta persistent cache failed: " << status; + return false; + } + return true; +} + } // namespace doris diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 0c62c963ce122d..a7c93496dde442 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -17,18 +17,79 @@ #pragma once +#include +#include +#include +#include + +#include "common/status.h" #include "io/file_factory.h" #include "io/fs/file_reader_writer_fwd.h" #include "util/obj_lru_cache.h" namespace doris { +enum class FileMetaCacheFormat : uint8_t { + PARQUET = 1, + ORC = 2, + PARQUET_V2 = 3, + ORC_V2 = 4, +}; + +struct FileMetaCacheContext { + FileMetaCacheFormat format; + const std::string& key; + int64_t modification_time = 0; + int64_t file_size = 0; + bool enable_memory_cache = true; +}; + +enum class FileMetaCacheLookupState { + MEMORY_HIT, + PERSISTED_HIT, + MISS, +}; + +struct FileMetaCacheLookupResult { + FileMetaCacheLookupState state = FileMetaCacheLookupState::MISS; +}; + +struct FileMetaCacheInsertResult { + bool memory_inserted = false; + bool persisted_inserted = false; +}; + +struct FileMetaCacheProfile { + int64_t* hit_cache = nullptr; + int64_t* hit_memory_cache = nullptr; + int64_t* hit_disk_cache = nullptr; + int64_t* miss_disk_cache = nullptr; + int64_t* write_disk_cache = nullptr; + int64_t* read_disk_cache_time = nullptr; + int64_t* write_disk_cache_time = nullptr; +}; + +class FileMetaPersistentCache { +public: + virtual ~FileMetaPersistentCache() = default; + + virtual Status read(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, std::string* payload) = 0; + + virtual Status write(FileMetaCacheFormat format, const std::string& key, + int64_t modification_time, int64_t file_size, + std::string_view payload) = 0; + + virtual void remove(FileMetaCacheFormat format, const std::string& key) = 0; +}; + // A file meta cache depends on a LRU cache. // Such as parsed parquet footer. // The capacity will limit the number of cache entries in cache. class FileMetaCache { public: - FileMetaCache(int64_t capacity) : _cache(capacity) {} + explicit FileMetaCache(int64_t capacity, + std::unique_ptr persistent_cache = nullptr); FileMetaCache(const FileMetaCache&) = delete; const FileMetaCache& operator=(const FileMetaCache&) = delete; @@ -40,6 +101,14 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); + static std::string get_memory_cache_key(FileMetaCacheFormat format, std::string_view key); + static std::string get_memory_cache_key(const FileMetaCacheContext& context) { + return get_memory_cache_key(context.format, context.key); + } + + static bool is_persistent_cache_configured(); + static bool is_persistent_cache_enabled(); + static bool is_persistent_cache_payload_size_allowed(uint64_t payload_size); bool lookup(const std::string& key, ObjLRUCache::CacheHandle* handle) { return _cache.lookup({key}, handle); @@ -50,10 +119,56 @@ class FileMetaCache { _cache.insert({key}, value, handle); } + template + bool insert(const std::string& key, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle) { + DCHECK(value != nullptr); + if (!_cache.enabled()) { + return false; + } + _cache.insert({key}, value.release(), handle); + return true; + } + bool enabled() const { return _cache.enabled(); } + FileMetaCacheLookupResult lookup(const FileMetaCacheContext& context, + ObjLRUCache::CacheHandle* handle, std::string* serialized_meta, + FileMetaCacheProfile* profile = nullptr); + + void invalidate_persistent_cache(const FileMetaCacheContext& context); + + template + FileMetaCacheInsertResult insert(const FileMetaCacheContext& context, std::unique_ptr& value, + ObjLRUCache::CacheHandle* handle, + std::string_view serialized_meta, + FileMetaCacheProfile* profile = nullptr) { + FileMetaCacheInsertResult result; + int64_t persisted_write_time = 0; + result.persisted_inserted = + insert_persistent_cache(context, serialized_meta, &persisted_write_time); + if (result.persisted_inserted && profile != nullptr) { + if (profile->write_disk_cache != nullptr) { + ++(*profile->write_disk_cache); + } + if (profile->write_disk_cache_time != nullptr) { + *profile->write_disk_cache_time += persisted_write_time; + } + } + if (context.enable_memory_cache) { + result.memory_inserted = insert(get_memory_cache_key(context), value, handle); + } + return result; + } + private: + bool lookup_persistent_cache(const FileMetaCacheContext& context, std::string* payload, + int64_t* read_time); + bool insert_persistent_cache(const FileMetaCacheContext& context, std::string_view payload, + int64_t* write_time); + ObjLRUCache _cache; + std::unique_ptr _persistent_cache; }; } // namespace doris diff --git a/be/src/io/fs/file_meta_disk_cache.cpp b/be/src/io/fs/file_meta_disk_cache.cpp new file mode 100644 index 00000000000000..7f196771cf0eaf --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.cpp @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/file_meta_disk_cache.h" + +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +namespace { + +constexpr std::string_view FILE_META_DISK_CACHE_MAGIC = "DFMC"; +constexpr uint8_t FILE_META_DISK_CACHE_VERSION = 1; +constexpr size_t FILE_META_DISK_CACHE_HEADER_SIZE = 4 + 1 + 1 + 2 + 8 + 8 + 8 + 4; + +std::string_view format_name(FileMetaCacheFormat format) { + switch (format) { + case FileMetaCacheFormat::PARQUET: + return "parquet"; + case FileMetaCacheFormat::ORC: + return "orc"; + case FileMetaCacheFormat::PARQUET_V2: + return "parquet_v2"; + case FileMetaCacheFormat::ORC_V2: + return "orc_v2"; + } + DCHECK(false) << "unknown file meta cache format"; + return "unknown"; +} + +Status parse_format(uint8_t format, FileMetaCacheFormat* parsed) { + switch (static_cast(format)) { + case FileMetaCacheFormat::PARQUET: + case FileMetaCacheFormat::ORC: + case FileMetaCacheFormat::PARQUET_V2: + case FileMetaCacheFormat::ORC_V2: + *parsed = static_cast(format); + return Status::OK(); + } + return Status::NotFound("file meta disk cache format mismatch"); +} + +struct FileMetaDiskCacheHeader { + FileMetaCacheFormat format; + int64_t modification_time = 0; + int64_t file_size = 0; + uint64_t payload_size = 0; + uint32_t checksum = 0; +}; + +Status parse_header(std::string_view header, FileMetaDiskCacheHeader* parsed) { + DCHECK(header.size() == FILE_META_DISK_CACHE_HEADER_SIZE); + if (std::memcmp(header.data(), FILE_META_DISK_CACHE_MAGIC.data(), + FILE_META_DISK_CACHE_MAGIC.size()) != 0) { + return Status::NotFound("file meta disk cache magic mismatch"); + } + + const auto* ptr = + reinterpret_cast(header.data() + FILE_META_DISK_CACHE_MAGIC.size()); + const uint8_t version = *ptr++; + if (version != FILE_META_DISK_CACHE_VERSION) { + return Status::NotFound("file meta disk cache version mismatch"); + } + + RETURN_IF_ERROR(parse_format(*ptr++, &parsed->format)); + ptr += 2; + parsed->file_size = static_cast(decode_fixed64_le(ptr)); + ptr += sizeof(uint64_t); + parsed->modification_time = static_cast(decode_fixed64_le(ptr)); + ptr += sizeof(uint64_t); + parsed->payload_size = decode_fixed64_le(ptr); + ptr += sizeof(uint64_t); + parsed->checksum = decode_fixed32_le(ptr); + return Status::OK(); +} + +std::string build_cache_value(FileMetaCacheFormat format, int64_t modification_time, + int64_t file_size, std::string_view payload) { + std::string value; + value.reserve(FILE_META_DISK_CACHE_HEADER_SIZE + payload.size()); + value.append(FILE_META_DISK_CACHE_MAGIC.data(), FILE_META_DISK_CACHE_MAGIC.size()); + value.push_back(static_cast(FILE_META_DISK_CACHE_VERSION)); + value.push_back(static_cast(format)); + value.push_back(0); + value.push_back(0); + put_fixed64_le(&value, static_cast(file_size)); + put_fixed64_le(&value, static_cast(modification_time)); + put_fixed64_le(&value, payload.size()); + put_fixed32_le(&value, crc32c::Crc32c(payload.data(), payload.size())); + value.append(payload.data(), payload.size()); + return value; +} + +io::CacheContext build_meta_cache_context() { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + return context; +} + +Status read_cached_range(io::BlockFileCache* cache, const io::UInt128Wrapper& hash, size_t offset, + size_t size, const io::CacheContext& context, std::string* output, + bool* missing_cached_range = nullptr) { + DCHECK(cache != nullptr); + DCHECK(output != nullptr); + output->clear(); + if (missing_cached_range != nullptr) { + *missing_cached_range = false; + } + if (size == 0) { + return Status::OK(); + } + + io::FileBlocks blocks; + bool fully_covered = false; + RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, + &blocks, &fully_covered)); + if (!fully_covered) { + if (missing_cached_range != nullptr) { + *missing_cached_range = true; + } + return Status::NotFound("file meta disk cache range is not cached"); + } + + output->resize(size); + size_t copied_size = 0; + const size_t requested_right = offset + size - 1; + for (const auto& block : blocks) { + const auto& range = block->range(); + const size_t read_left = std::max(range.left, offset); + const size_t read_right = std::min(range.right, requested_right); + DCHECK_LE(read_left, read_right); + const size_t read_size = read_right - read_left + 1; + RETURN_IF_ERROR(block->read(Slice(output->data() + copied_size, read_size), + read_left - range.left)); + copied_size += read_size; + } + DCHECK_EQ(copied_size, size); + return Status::OK(); +} + +} // namespace + +std::string FileMetaDiskCache::get_key(FileMetaCacheFormat format, + std::string_view file_meta_cache_key) { + std::string key; + key.reserve(32 + file_meta_cache_key.size()); + key.append("file_meta_cache:v1:"); + key.append(format_name(format)); + key.push_back(':'); + key.append(file_meta_cache_key.data(), file_meta_cache_key.size()); + return key; +} + +Status FileMetaDiskCache::read(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, std::string* payload) { + payload->clear(); + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + + auto invalidate_entry = [&](const Status& status) { + payload->clear(); + cache->remove_if_cached(hash); + return status; + }; + + std::string header; + bool missing_header_range = false; + Status status = read_cached_range(cache, hash, 0, FILE_META_DISK_CACHE_HEADER_SIZE, context, + &header, &missing_header_range); + if (!status.ok()) { + if (missing_header_range) { + return status; + } + return invalidate_entry(status); + } + + FileMetaDiskCacheHeader parsed; + status = parse_header(header, &parsed); + if (!status.ok()) { + return invalidate_entry(status); + } + if (parsed.format != format || parsed.modification_time != modification_time || + parsed.file_size != file_size || + !FileMetaCache::is_persistent_cache_payload_size_allowed(parsed.payload_size)) { + return invalidate_entry(Status::NotFound("file meta disk cache header mismatch")); + } + + bool missing_payload_range = false; + status = read_cached_range(cache, hash, FILE_META_DISK_CACHE_HEADER_SIZE, parsed.payload_size, + context, payload, &missing_payload_range); + if (!status.ok()) { + if (missing_payload_range) { + payload->clear(); + return status; + } + return invalidate_entry(status); + } + const uint32_t checksum = crc32c::Crc32c(payload->data(), payload->size()); + if (checksum != parsed.checksum) { + return invalidate_entry(Status::NotFound("file meta disk cache checksum mismatch")); + } + return Status::OK(); +} + +Status FileMetaDiskCache::write(FileMetaCacheFormat format, const std::string& file_meta_cache_key, + int64_t modification_time, int64_t file_size, + std::string_view payload) { + if (!FileMetaCache::is_persistent_cache_payload_size_allowed( + static_cast(payload.size()))) { + return Status::NotFound("file meta disk cache payload size is not allowed"); + } + + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache == nullptr) { + return Status::NotFound("file meta disk cache is not available"); + } + + const std::string value = build_cache_value(format, modification_time, file_size, payload); + io::ReadStatistics stats; + io::CacheContext context = build_meta_cache_context(); + context.stats = &stats; + auto holder = cache->get_or_set(hash, 0, value.size(), context); + + // Claim the complete entry before appending so a rejected block cannot leave partial data. + io::FileBlocks blocks_to_write; + for (const auto& block : holder.file_blocks) { + auto state = block->state(); + if (state == io::FileBlock::State::DOWNLOADING && !block->is_downloader()) { + state = block->wait(); + } + if (state == io::FileBlock::State::DOWNLOADED) { + continue; + } + if (state != io::FileBlock::State::EMPTY) { + return Status::NotFound("file meta disk cache block is not writable"); + } + if (block->get_or_set_downloader() != io::FileBlock::get_caller_id()) { + return Status::NotFound("file meta disk cache block has another downloader"); + } + blocks_to_write.emplace_back(block); + } + + auto write_block = [&](const io::FileBlockSPtr& block) -> Status { + const auto& range = block->range(); + DCHECK_LT(range.right, value.size()); + Status status = block->append(Slice(value.data() + range.left, range.size())); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + status = block->finalize(); + if (!status.ok()) { + cache->remove_if_cached(hash); + return status; + } + return Status::OK(); + }; + + for (const auto& block : blocks_to_write) { + if (block->range().left >= FILE_META_DISK_CACHE_HEADER_SIZE) { + RETURN_IF_ERROR(write_block(block)); + } + } + for (const auto& block : blocks_to_write) { + if (block->range().left < FILE_META_DISK_CACHE_HEADER_SIZE) { + RETURN_IF_ERROR(write_block(block)); + } + } + return Status::OK(); +} + +void FileMetaDiskCache::remove(FileMetaCacheFormat format, const std::string& file_meta_cache_key) { + const std::string cache_key = get_key(format, file_meta_cache_key); + const auto hash = io::BlockFileCache::hash(cache_key); + io::BlockFileCache* cache = get_cache(hash); + if (cache != nullptr) { + cache->remove_if_cached(hash); + } +} + +io::BlockFileCache* FileMetaDiskCache::get_cache(const io::UInt128Wrapper& hash) const { + if (_cache != nullptr) { + return _cache; + } + io::FileCacheFactory* factory = io::FileCacheFactory::instance(); + if (factory == nullptr || factory->get_cache_instance_size() == 0) { + return nullptr; + } + return factory->get_by_path(hash); +} + +} // namespace doris diff --git a/be/src/io/fs/file_meta_disk_cache.h b/be/src/io/fs/file_meta_disk_cache.h new file mode 100644 index 00000000000000..e844690764a3b0 --- /dev/null +++ b/be/src/io/fs/file_meta_disk_cache.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "io/fs/file_meta_cache.h" + +namespace doris { + +namespace io { +class BlockFileCache; +struct UInt128Wrapper; +} // namespace io + +class FileMetaDiskCache final : public FileMetaPersistentCache { +public: + explicit FileMetaDiskCache(io::BlockFileCache* cache = nullptr) : _cache(cache) {} + + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override; + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override; + + void remove(FileMetaCacheFormat format, const std::string& key) override; + +private: + static std::string get_key(FileMetaCacheFormat format, std::string_view file_meta_cache_key); + + io::BlockFileCache* get_cache(const io::UInt128Wrapper& hash) const; + + io::BlockFileCache* _cache = nullptr; +}; + +} // namespace doris diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index c010c1deeba272..a5121d29975c9b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -62,6 +62,7 @@ #include "io/cache/block_file_cache_factory.h" #include "io/cache/fs_file_cache_storage.h" #include "io/fs/file_meta_cache.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/local_file_reader.h" #include "load/channel/load_channel_mgr.h" #include "load/channel/load_stream_mgr.h" @@ -506,7 +507,9 @@ void ExecEnv::init_file_cache_factory(std::vector& cache_paths "= true"; exit(-1); } - return; + if (!FileMetaCache::is_persistent_cache_configured()) { + return; + } } if (config::file_cache_each_block_size > config::s3_write_buffer_size || config::s3_write_buffer_size % config::file_cache_each_block_size != 0) { @@ -676,7 +679,10 @@ Status ExecEnv::init_mem_env() { << config::file_cache_max_file_reader_cache_size; config::file_cache_max_file_reader_cache_size = block_file_cache_fd_cache_size; - _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num); + _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num, + FileMetaCache::is_persistent_cache_configured() + ? std::make_unique() + : nullptr); _lookup_connection_cache = LookupConnectionCache::create_global_instance(config::lookup_connection_cache_capacity); diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 4f8f803bd3ef28..f54a07aba02845 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -17,9 +17,26 @@ #include "io/fs/file_meta_cache.h" +#include +#include + +#include +#include +#include +#include + +#include "common/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/file_block.h" +#include "io/cache/file_cache_common.h" +#include "io/cache/fs_file_cache_storage.h" +#include "io/fs/file_meta_disk_cache.h" #include "io/fs/file_reader.h" +#include "runtime/exec_env.h" +#include "util/coding.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -27,7 +44,7 @@ namespace { class MockFileReader : public io::FileReader { public: MockFileReader(const std::string& file_name, size_t size) - : _file_name(file_name), _size(size), _closed(false) {} + : _file_name(file_name), _size(size) {} ~MockFileReader() override = default; const io::Path& path() const override { @@ -56,8 +73,127 @@ class MockFileReader : public io::FileReader { private: std::string _file_name; size_t _size; - bool _closed; + bool _closed = false; }; + +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++read_count; + if (fail_read) { + return Status::IOError("injected persistent cache read failure"); + } + if (!has_value || format != stored_format || key != stored_key || + modification_time != stored_modification_time || file_size != stored_file_size) { + return Status::NotFound("persistent cache entry not found"); + } + *payload = stored_payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++write_count; + if (fail_write) { + return Status::IOError("injected persistent cache write failure"); + } + stored_format = format; + stored_key = key; + stored_modification_time = modification_time; + stored_file_size = file_size; + stored_payload = payload; + has_value = true; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + ++remove_count; + if (has_value && format == stored_format && key == stored_key) { + has_value = false; + } + } + + bool fail_read = false; + bool fail_write = false; + bool has_value = false; + int read_count = 0; + int write_count = 0; + int remove_count = 0; + FileMetaCacheFormat stored_format = FileMetaCacheFormat::PARQUET; + std::string stored_key; + int64_t stored_modification_time = 0; + int64_t stored_file_size = 0; + std::string stored_payload; +}; + +io::FileCacheSettings create_file_meta_disk_cache_test_settings() { + io::FileCacheSettings settings; + settings.capacity = 1024 * 1024; + settings.max_file_block_size = 8; + settings.index_queue_size = 1024 * 1024; + settings.index_queue_elements = 1024; + settings.max_query_cache_size = 1024 * 1024; + settings.storage = "memory"; + return settings; +} + +constexpr size_t FILE_META_DISK_CACHE_TEST_HEADER_SIZE = 36; + +std::string file_meta_disk_cache_key(FileMetaCacheFormat format, const std::string& meta_key) { + std::string key = "file_meta_cache:v1:"; + switch (format) { + case FileMetaCacheFormat::PARQUET: + key.append("parquet"); + break; + case FileMetaCacheFormat::ORC: + key.append("orc"); + break; + case FileMetaCacheFormat::PARQUET_V2: + key.append("parquet_v2"); + break; + case FileMetaCacheFormat::ORC_V2: + key.append("orc_v2"); + break; + } + key.push_back(':'); + key.append(meta_key); + return key; +} + +std::string build_file_meta_disk_cache_value(FileMetaCacheFormat format, int64_t modification_time, + int64_t file_size, std::string_view payload) { + std::string value; + value.append("DFMC", 4); + value.push_back(1); + value.push_back(static_cast(format)); + value.push_back(0); + value.push_back(0); + put_fixed64_le(&value, static_cast(file_size)); + put_fixed64_le(&value, static_cast(modification_time)); + put_fixed64_le(&value, payload.size()); + put_fixed32_le(&value, crc32c::Crc32c(payload.data(), payload.size())); + value.append(payload); + return value; +} + +io::CacheContext create_file_meta_disk_cache_context(io::ReadStatistics* stats) { + io::CacheContext context; + context.cache_type = io::FileCacheType::INDEX; + context.query_id = TUniqueId(); + context.expiration_time = 0; + context.is_cold_data = false; + context.is_warmup = false; + context.stats = stats; + return context; +} + +bool wait_until_file_cache_ready(io::BlockFileCache* cache) { + for (int i = 0; i < 1000 && !cache->get_async_open_success(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return cache->get_async_open_success(); +} } // anonymous namespace TEST(FileMetaCacheTest, KeyGenerationFromParams) { @@ -73,7 +209,11 @@ TEST(FileMetaCacheTest, KeyGenerationFromParams) { std::string key3 = FileMetaCache::get_key(file_name, mtime + 1, file_size); EXPECT_NE(key1, key3); - // mtime == 0, use file_size + // Same mtime with different size should produce different key + std::string key_with_different_size = FileMetaCache::get_key(file_name, mtime, file_size + 1); + EXPECT_NE(key1, key_with_different_size); + + // mtime == 0, still includes file_size std::string key4 = FileMetaCache::get_key(file_name, 0, file_size); std::string key5 = FileMetaCache::get_key(file_name, 0, file_size); EXPECT_EQ(key4, key5); @@ -116,19 +256,27 @@ TEST(FileMetaCacheTest, KeyContentVerification) { std::string key_with_mtime = FileMetaCache::get_key(file_name, mtime, file_size); - ASSERT_EQ(key_with_mtime.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_mtime.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_mtime.data(), file_name.data(), file_name.size()), 0); int64_t extracted_mtime = 0; memcpy(&extracted_mtime, key_with_mtime.data() + file_name.size(), sizeof(int64_t)); EXPECT_EQ(extracted_mtime, mtime); + int64_t extracted_file_size = 0; + memcpy(&extracted_file_size, key_with_mtime.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); + EXPECT_EQ(extracted_file_size, file_size); std::string key_with_filesize = FileMetaCache::get_key(file_name, 0, file_size); - ASSERT_EQ(key_with_filesize.size(), file_name.size() + sizeof(int64_t)); + ASSERT_EQ(key_with_filesize.size(), file_name.size() + 2 * sizeof(int64_t)); EXPECT_EQ(memcmp(key_with_filesize.data(), file_name.data(), file_name.size()), 0); + int64_t extracted_zero_mtime = -1; + memcpy(&extracted_zero_mtime, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + EXPECT_EQ(extracted_zero_mtime, 0); int64_t extracted_filesize = 0; - memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size(), sizeof(int64_t)); + memcpy(&extracted_filesize, key_with_filesize.data() + file_name.size() + sizeof(int64_t), + sizeof(int64_t)); EXPECT_EQ(extracted_filesize, file_size); } @@ -155,4 +303,696 @@ TEST(FileMetaCacheTest, InsertAndLookupWithIntValue) { EXPECT_EQ(*cached_val2, 12345); } -} // namespace doris \ No newline at end of file +TEST(FileMetaCacheTest, ContextLookupReportsMemoryHit) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE(cache.insert(FileMetaCache::get_memory_cache_key(context), value, &insert_handle)); + ASSERT_TRUE(insert_handle.valid()); + + int64_t hit_cache = 0; + int64_t hit_memory_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_memory_cache = &hit_memory_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache}; + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + const FileMetaCacheLookupResult result = + cache.lookup(context, &lookup_handle, &serialized_meta, &profile); + + EXPECT_EQ(result.state, FileMetaCacheLookupState::MEMORY_HIT); + EXPECT_TRUE(lookup_handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_memory_cache, 1); + EXPECT_EQ(hit_disk_cache, 0); + EXPECT_EQ(miss_disk_cache, 0); +} + +TEST(FileMetaCacheTest, ContextLookupClearsReusedMemoryHandle) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(1024, std::move(persistent_cache)); + + const std::string memory_key = + FileMetaCache::get_key("s3://bucket/memory-handle.parquet", 123, 456); + const FileMetaCacheContext memory_context {.format = FileMetaCacheFormat::PARQUET, + .key = memory_key, + .modification_time = 123, + .file_size = 456}; + auto memory_value = std::make_unique("parsed memory footer"); + ObjLRUCache::CacheHandle insert_handle; + ASSERT_TRUE( + cache.insert(memory_context, memory_value, &insert_handle, "serialized memory footer") + .memory_inserted); + + ObjLRUCache::CacheHandle reused_handle; + std::string serialized_meta; + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + + const std::string persistent_key = + FileMetaCache::get_key("s3://bucket/persistent-handle.parquet", 123, 456); + const FileMetaCacheContext persistent_context {.format = FileMetaCacheFormat::PARQUET, + .key = persistent_key, + .modification_time = 123, + .file_size = 456}; + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = persistent_context.format; + persistent_cache_ptr->stored_key = persistent_context.key; + persistent_cache_ptr->stored_modification_time = persistent_context.modification_time; + persistent_cache_ptr->stored_file_size = persistent_context.file_size; + persistent_cache_ptr->stored_payload = "serialized persistent footer"; + + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_FALSE(reused_handle.valid()); + + ASSERT_EQ(cache.lookup(memory_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); + ASSERT_TRUE(reused_handle.valid()); + persistent_cache_ptr->has_value = false; + EXPECT_EQ(cache.lookup(persistent_context, &reused_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(reused_handle.valid()); +} + +TEST(FileMetaCacheTest, ContextMemoryCacheKeyIncludesFormat) { + FileMetaCache cache(1024 * 1024); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.parquet", 123, 456); + const FileMetaCacheContext parquet_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + const FileMetaCacheContext parquet_v2_context {.format = FileMetaCacheFormat::PARQUET_V2, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = true}; + + auto value = std::make_unique(7); + ObjLRUCache::CacheHandle insert_handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(parquet_context, value, &insert_handle, "payload"); + ASSERT_TRUE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string serialized_meta; + EXPECT_EQ(cache.lookup(parquet_v2_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(cache.lookup(parquet_context, &lookup_handle, &serialized_meta).state, + FileMetaCacheLookupState::MEMORY_HIT); +} + +TEST(FileMetaCacheTest, PersistentCacheConfigurationIsIndependentFromRuntimeSizeGate) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 0; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); + + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + EXPECT_TRUE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_TRUE(FileMetaCache::is_persistent_cache_enabled()); + + config::enable_external_file_meta_disk_cache = false; + EXPECT_FALSE(FileMetaCache::is_persistent_cache_configured()); + EXPECT_FALSE(FileMetaCache::is_persistent_cache_enabled()); +} + +TEST(FileMetaCacheDiskTest, OrcFormatsUseIndependentKeys) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_orc_format_isolation_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/same.orc", 123, 456); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC, meta_key, 123, 456, + "legacy orc serialized tail") + .ok()); + ASSERT_TRUE(disk_cache + .write(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, + "format v2 orc serialized tail") + .ok()); + + std::string legacy_payload; + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::ORC, meta_key, 123, 456, &legacy_payload).ok()); + EXPECT_EQ(legacy_payload, "legacy orc serialized tail"); + + std::string v2_payload; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::ORC_V2, meta_key, 123, 456, &v2_payload).ok()); + EXPECT_EQ(v2_payload, "format v2 orc serialized tail"); +} + +TEST(FileMetaCacheTest, ContextInsertCanSkipMemoryCacheWithoutPersistentStore) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = false; + + FileMetaCache cache(1024 * 1024); + const std::string meta_key = + FileMetaCache::get_key("s3://bucket/skip-memory.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult result = + cache.insert(context, value, &handle, std::string_view(*value)); + + EXPECT_FALSE(result.persisted_inserted); + EXPECT_FALSE(result.memory_inserted); + EXPECT_FALSE(handle.valid()); + EXPECT_NE(value, nullptr); + + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_FALSE(cache.lookup(meta_key, &lookup_handle)); +} + +TEST(FileMetaCacheTest, PersistentCacheRoundTripUsesFileMetaBoundary) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/persistent.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto parsed_value = std::make_unique("parsed footer"); + ObjLRUCache::CacheHandle handle; + int64_t hit_cache = 0; + int64_t hit_disk_cache = 0; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.hit_cache = &hit_cache, + .hit_disk_cache = &hit_disk_cache, + .miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + ASSERT_TRUE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 1); + + std::string serialized_meta; + const FileMetaCacheLookupResult lookup_result = + cache.lookup(context, &handle, &serialized_meta, &profile); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(serialized_meta, "serialized footer"); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(hit_cache, 1); + EXPECT_EQ(hit_disk_cache, 1); + EXPECT_EQ(miss_disk_cache, 0); + + cache.invalidate_persistent_cache(context); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 1); +} + +TEST(FileMetaCacheTest, PersistentCacheFailureFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->fail_read = true; + persistent_cache_ptr->fail_write = true; + FileMetaCache cache(0, std::move(persistent_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/failure.orc", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::ORC, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + int64_t miss_disk_cache = 0; + int64_t write_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache, + .write_disk_cache = &write_disk_cache}; + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(miss_disk_cache, 1); + + auto parsed_value = std::make_unique("parsed footer"); + const FileMetaCacheInsertResult insert_result = + cache.insert(context, parsed_value, &handle, "serialized footer", &profile); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + EXPECT_NE(parsed_value, nullptr); + EXPECT_EQ(persistent_cache_ptr->write_count, 1); + EXPECT_EQ(write_disk_cache, 0); +} + +TEST(FileMetaCacheTest, OversizedPersistentReadFallsBackToMiss) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 4; + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/oversized.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + auto persistent_cache = std::make_unique(); + auto* persistent_cache_ptr = persistent_cache.get(); + persistent_cache_ptr->has_value = true; + persistent_cache_ptr->stored_format = context.format; + persistent_cache_ptr->stored_key = context.key; + persistent_cache_ptr->stored_modification_time = context.modification_time; + persistent_cache_ptr->stored_file_size = context.file_size; + persistent_cache_ptr->stored_payload = "oversized payload"; + FileMetaCache cache(0, std::move(persistent_cache)); + + ObjLRUCache::CacheHandle handle; + std::string serialized_meta = "stale output"; + EXPECT_EQ(cache.lookup(context, &handle, &serialized_meta).state, + FileMetaCacheLookupState::MISS); + EXPECT_FALSE(handle.valid()); + EXPECT_TRUE(serialized_meta.empty()); + EXPECT_EQ(persistent_cache_ptr->read_count, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count, 1); + EXPECT_FALSE(persistent_cache_ptr->has_value); +} + +TEST(FileMetaCacheDiskTest, PersistentCacheRequiresStableModificationTime) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_unstable_identity_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/mtime-zero.parquet", 0, 456); + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(0, std::move(disk_cache)); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 0, + .file_size = 456, + .enable_memory_cache = false}; + + auto value = std::make_unique("serialized footer payload"); + ObjLRUCache::CacheHandle handle; + const FileMetaCacheInsertResult insert_result = + cache.insert(context, value, &handle, std::string_view(*value)); + EXPECT_FALSE(insert_result.persisted_inserted); + EXPECT_FALSE(insert_result.memory_inserted); + + int64_t miss_disk_cache = 0; + FileMetaCacheProfile profile {.miss_disk_cache = &miss_disk_cache}; + std::string serialized_meta; + ObjLRUCache::CacheHandle lookup_handle; + EXPECT_EQ(cache.lookup(context, &lookup_handle, &serialized_meta, &profile).state, + FileMetaCacheLookupState::MISS); + EXPECT_EQ(miss_disk_cache, 0); +} + +TEST(FileMetaCacheDiskTest, PersistentCacheStoresPayloadBehindFileMetaInterface) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_interface_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + auto disk_cache = std::make_unique(&block_cache); + FileMetaCache cache(1024 * 1024, std::move(disk_cache)); + const std::string meta_key = FileMetaCache::get_key("s3://bucket/interface.parquet", 123, 456); + const FileMetaCacheContext context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 123, + .file_size = 456, + .enable_memory_cache = false}; + const std::string payload = "serialized footer payload"; + auto value = std::make_unique(payload); + ObjLRUCache::CacheHandle handle; + + const FileMetaCacheInsertResult insert_result = cache.insert(context, value, &handle, payload); + ASSERT_TRUE(insert_result.persisted_inserted); + ASSERT_FALSE(insert_result.memory_inserted); + + ObjLRUCache::CacheHandle lookup_handle; + std::string output; + const FileMetaCacheLookupResult lookup_result = cache.lookup(context, &lookup_handle, &output); + EXPECT_EQ(lookup_result.state, FileMetaCacheLookupState::PERSISTED_HIT); + EXPECT_EQ(output, payload); + EXPECT_FALSE(lookup_handle.valid()); + + const FileMetaCacheContext stale_context {.format = FileMetaCacheFormat::PARQUET, + .key = meta_key, + .modification_time = 124, + .file_size = 456, + .enable_memory_cache = false}; + output.clear(); + const FileMetaCacheLookupResult stale_lookup_result = + cache.lookup(stale_context, &lookup_handle, &output); + EXPECT_EQ(stale_lookup_result.state, FileMetaCacheLookupState::MISS); + EXPECT_TRUE(output.empty()); +} + +TEST(FileMetaCacheDiskTest, CapacityFailureDoesNotLeavePartiallyDownloadedBlocks) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.capacity = 64; + settings.index_queue_size = 64; + settings.index_queue_elements = 8; + settings.max_query_cache_size = 0; + io::BlockFileCache block_cache("file_meta_disk_cache_capacity_failure_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/too-large.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, + std::string(80, 'p')); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(block_cache.get_blocks_by_key(hash).empty()); +} + +TEST(FileMetaCacheDiskTest, DiskEntrySurvivesBlockFileCacheReinitialization) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + const int64_t old_file_cache_background_ttl_gc_interval_ms = + config::file_cache_background_ttl_gc_interval_ms; + const int64_t old_file_cache_background_ttl_info_update_interval_ms = + config::file_cache_background_ttl_info_update_interval_ms; + Defer restore_config {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + config::file_cache_background_ttl_gc_interval_ms = + old_file_cache_background_ttl_gc_interval_ms; + config::file_cache_background_ttl_info_update_interval_ms = + old_file_cache_background_ttl_info_update_interval_ms; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1024; + config::file_cache_background_ttl_gc_interval_ms = 20; + config::file_cache_background_ttl_info_update_interval_ms = 20; + + const std::filesystem::path cache_path = + std::filesystem::temp_directory_path() / + ("doris_file_meta_disk_cache_restart_" + std::to_string(getpid())); + std::error_code error; + std::filesystem::remove_all(cache_path, error); + ASSERT_TRUE(std::filesystem::create_directories(cache_path)); + Defer remove_cache_path {[&] { std::filesystem::remove_all(cache_path, error); }}; + + auto* exec_env = ExecEnv::GetInstance(); + const bool owns_fd_cache = exec_env->file_cache_open_fd_cache() == nullptr; + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(std::make_unique()); + } + Defer release_fd_cache {[&] { + if (owns_fd_cache) { + exec_env->set_file_cache_open_fd_cache(nullptr); + } + }}; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + settings.storage = "disk"; + const std::string meta_key = FileMetaCache::get_key("s3://bucket/restart.parquet", 123, 456); + const std::string payload = "serialized footer persisted across cache initialization"; + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + ASSERT_TRUE( + disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, payload).ok()); + } + + { + io::BlockFileCache block_cache(cache_path.string(), settings); + ASSERT_TRUE(block_cache.initialize().ok()); + ASSERT_TRUE(wait_until_file_cache_ready(&block_cache)); + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + ASSERT_TRUE( + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); + } +} + +TEST(FileMetaCacheDiskTest, IncompleteMultiBlockEntryMissesWithoutInvalidating) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_incomplete_entry_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/incomplete.parquet", 123, 456); + const std::string payload = "serialized footer payload spanning several blocks"; + const std::string value = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto holder = block_cache.get_or_set(hash, 0, value.size(), context); + bool covered_header = false; + for (const auto& block : holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + continue; + } + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + covered_header = block->range().right + 1 >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE; + } + ASSERT_TRUE(covered_header); + } + + FileMetaDiskCache disk_cache(&block_cache); + std::string output; + const Status status = + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(output.empty()); + EXPECT_FALSE(block_cache.get_blocks_by_key(hash).empty()); +} + +TEST(FileMetaCacheDiskTest, PayloadMissDoesNotRemoveInFlightWriterBlocks) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_inflight_writer_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/inflight.parquet", 123, 456); + const std::string payload = "serialized footer payload spanning several blocks"; + const std::string value = + build_file_meta_disk_cache_value(FileMetaCacheFormat::PARQUET, 123, 456, payload); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto partial_holder = block_cache.get_or_set(hash, 0, value.size(), context); + for (const auto& block : partial_holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + continue; + } + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + } + } + + FileMetaDiskCache disk_cache(&block_cache); + { + io::ReadStatistics stats; + io::CacheContext context = create_file_meta_disk_cache_context(&stats); + auto writer_holder = block_cache.get_or_set(hash, 0, value.size(), context); + bool has_inflight_payload_block = false; + for (const auto& block : writer_holder.file_blocks) { + if (block->range().left >= FILE_META_DISK_CACHE_TEST_HEADER_SIZE) { + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + has_inflight_payload_block = true; + break; + } + } + ASSERT_TRUE(has_inflight_payload_block); + + std::string output; + const Status read_status = + disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output); + EXPECT_FALSE(read_status.ok()); + EXPECT_TRUE(output.empty()); + + for (const auto& block : writer_holder.file_blocks) { + auto state = block->state(); + if (state == io::FileBlock::State::DOWNLOADED) { + continue; + } + if (state == io::FileBlock::State::EMPTY) { + ASSERT_EQ(block->get_or_set_downloader(), io::FileBlock::get_caller_id()); + } else { + ASSERT_EQ(state, io::FileBlock::State::DOWNLOADING); + ASSERT_TRUE(block->is_downloader()); + } + ASSERT_TRUE( + block->append(Slice(value.data() + block->range().left, block->range().size())) + .ok()); + ASSERT_TRUE(block->finalize().ok()); + } + } + + std::string output; + ASSERT_TRUE(disk_cache.read(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, &output).ok()); + EXPECT_EQ(output, payload); +} + +TEST(FileMetaCacheDiskTest, EmptyPayloadIsNotPersisted) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + io::FileCacheSettings settings = create_file_meta_disk_cache_test_settings(); + io::BlockFileCache block_cache("file_meta_disk_cache_empty_payload_test", settings); + ASSERT_TRUE(block_cache.initialize().ok()); + + const std::string meta_key = FileMetaCache::get_key("s3://bucket/empty.parquet", 123, 456); + const auto hash = io::BlockFileCache::hash( + file_meta_disk_cache_key(FileMetaCacheFormat::PARQUET, meta_key)); + FileMetaDiskCache disk_cache(&block_cache); + + const Status status = disk_cache.write(FileMetaCacheFormat::PARQUET, meta_key, 123, 456, ""); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(block_cache.get_blocks_by_key(hash).empty()); +} + +} // namespace doris diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 89bf367d06b053..38152d94305476 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -845,6 +845,50 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } +TEST_F(IcebergReaderTest, position_delete_inherits_disabled_file_meta_memory_cache) { + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path("data.parquet"); + scan_range.__set_start_offset(0); + scan_range.__set_size(0); + + RuntimeProfile profile("test_profile"); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + + IcebergParquetReader iceberg_reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, + &io_ctx, &runtime_state, cache.get()); + iceberg_reader.set_enable_file_meta_memory_cache(false); + + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(IcebergReaderMixin::POSITION_DELETE); + delete_file.__set_path(mixed_position_delete_file()); + ASSERT_TRUE(iceberg_reader + .TEST_position_delete_base("file:///tmp/non_matching_data.parquet", + {delete_file}) + .ok()); + + io::FileReaderSPtr delete_file_reader; + ASSERT_TRUE(io::global_local_filesystem() + ->open_file(mixed_position_delete_file(), &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET, meta_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(cache->lookup(memory_key, &handle)); +} + // Test reading real Iceberg Orc file using IcebergTableReader TEST_F(IcebergReaderTest, read_iceberg_orc_file) { // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 0e36498c41d9bf..33c6499117ebf2 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_array.h" @@ -69,11 +71,13 @@ #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -4235,6 +4239,67 @@ format::LocalColumnIndex struct_child_projection(int32_t root_field_id, int32_t return projection; } +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + ++_read_count; + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + ++_write_count; + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int read_count() const { return _read_count; } + int write_count() const { return _write_count; } + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + return std::to_string(static_cast(format)) + ":" + key; + } + + std::map _entries; + int _read_count = 0; + int _write_count = 0; + int _remove_count = 0; +}; + class NewOrcReaderTest : public testing::Test { protected: void SetUp() override { @@ -4249,14 +4314,18 @@ class NewOrcReaderTest : public testing::Test { std::unique_ptr create_reader( RuntimeProfile* profile = nullptr, - std::optional global_rowid_context = std::nullopt) const { + std::optional global_rowid_context = std::nullopt, + FileMetaCache* file_meta_cache = nullptr, bool enable_file_meta_memory_cache = true, + int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); - return std::make_unique(system_properties, file_description, - nullptr, profile, global_rowid_context); + file_description->mtime = mtime; + return std::make_unique( + system_properties, file_description, nullptr, profile, global_rowid_context, false, + file_meta_cache, enable_file_meta_memory_cache); } std::unique_ptr create_reader_for_path( @@ -4295,6 +4364,189 @@ class NewOrcReaderTest : public testing::Test { std::string _file_path; }; +TEST_F(NewOrcReaderTest, UsesFileMetaCacheForSerializedTail) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); + + RuntimeProfile second_profile("new_orc_reader_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_orc_reader_persistent_file_meta_cache_first"); + auto first_reader = create_reader(&first_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_miss_disk_cache, 1); + EXPECT_EQ(first_reader->reader_statistics().file_footer_write_disk_cache, 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_FALSE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC, file_meta_cache_key)); + + RuntimeProfile second_profile("new_orc_reader_persistent_file_meta_cache_second"); + auto second_reader = create_reader(&second_profile, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_cache, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); +} + +TEST_F(NewOrcReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::ORC_V2, file_meta_cache_key, 123, file_size, + "not a serialized orc file tail") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + EXPECT_EQ(reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(reader->reader_statistics().file_footer_write_disk_cache, 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE(persistent_cache_ptr->contains(FileMetaCacheFormat::ORC_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::ORC_V2, file_meta_cache_key), + "not a serialized orc file tail"); +} + +TEST_F(NewOrcReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::ORC_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_disk_cache, 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + +TEST_F(NewOrcReaderTest, UnstableMtimeSkipsPersistentFileMetaCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + }}; + config::enable_external_file_meta_disk_cache = true; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(first_reader->init(&state).ok()); + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache, false, 0); + ASSERT_TRUE(second_reader->init(&state).ok()); + + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->read_count(), 0); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + +TEST_F(NewOrcReaderTest, OversizedPersistentPayloadRemainsMemoryCacheEligible) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 1; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(1024 * 1024, std::move(persistent_cache)); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto first_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + EXPECT_EQ(first_reader->reader_statistics().file_footer_read_calls, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); + + auto second_reader = create_reader(nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + EXPECT_EQ(second_reader->reader_statistics().file_footer_read_calls, 0); + EXPECT_EQ(second_reader->reader_statistics().file_footer_hit_memory_cache, 1); + EXPECT_EQ(persistent_cache_ptr->write_count(), 0); +} + TEST_F(NewOrcReaderTest, AggregatePushdownReturnsCountFromFileMetadata) { auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 6269a9fc2a7245..1b22419a80322c 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -35,6 +35,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_nullable.h" @@ -60,12 +61,14 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" +#include "io/fs/file_meta_cache.h" #include "io/io_common.h" #include "runtime/runtime_state.h" #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -1020,6 +1023,66 @@ class TestFileReader final : public format::FileReader { long io_context_use_count() const { return _io_ctx.use_count(); } }; +class TestPersistentFileMetaCache final : public FileMetaPersistentCache { +public: + Status read(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string* payload) override { + const auto it = _entries.find(entry_key(format, key)); + if (it == _entries.end()) { + return Status::NotFound("test file meta cache entry not found"); + } + if (it->second.modification_time != modification_time || + it->second.file_size != file_size) { + return Status::NotFound("test file meta cache entry stale"); + } + *payload = it->second.payload; + return Status::OK(); + } + + Status write(FileMetaCacheFormat format, const std::string& key, int64_t modification_time, + int64_t file_size, std::string_view payload) override { + _entries[entry_key(format, key)] = Entry {.modification_time = modification_time, + .file_size = file_size, + .payload = std::string(payload)}; + return Status::OK(); + } + + void remove(FileMetaCacheFormat format, const std::string& key) override { + _entries.erase(entry_key(format, key)); + ++_remove_count; + } + + bool contains(FileMetaCacheFormat format, const std::string& key) const { + return _entries.contains(entry_key(format, key)); + } + + std::string payload(FileMetaCacheFormat format, const std::string& key) const { + const auto it = _entries.find(entry_key(format, key)); + return it == _entries.end() ? std::string {} : it->second.payload; + } + + int remove_count() const { return _remove_count; } + +private: + struct Entry { + int64_t modification_time = 0; + int64_t file_size = 0; + std::string payload; + }; + + static std::string entry_key(FileMetaCacheFormat format, const std::string& key) { + std::string result; + result.reserve(key.size() + 4); + result.append(std::to_string(static_cast(format))); + result.push_back(':'); + result.append(key); + return result; + } + + std::map _entries; + int _remove_count = 0; +}; + TEST(FileReaderTest, OpenStoresRequestAndCloseKeepsRequest) { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; @@ -1073,17 +1136,21 @@ class NewParquetReaderTest : public testing::Test { int64_t range_start_offset = 0, int64_t range_size = -1, RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, - std::optional global_rowid_context = std::nullopt) const { + std::optional global_rowid_context = std::nullopt, + FileMetaCache* file_meta_cache = nullptr, bool enable_file_meta_memory_cache = true, + int64_t mtime = 123) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); file_description->path = _file_path; file_description->file_size = static_cast(std::filesystem::file_size(_file_path)); + file_description->mtime = mtime; file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, file_meta_cache, + enable_file_meta_memory_cache); } std::filesystem::path _test_dir; @@ -1108,6 +1175,182 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, UsesFileMetaCacheForFooterMetadata) { + FileMetaCache file_meta_cache(1024 * 1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_file_meta_cache_first"); + auto first_reader = + create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_TRUE(file_meta_cache.lookup(memory_cache_key, &handle)); + EXPECT_TRUE(handle.valid()); + + RuntimeProfile second_profile("new_parquet_reader_file_meta_cache_second"); + auto second_reader = + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 0); + std::vector schema; + ASSERT_TRUE(second_reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); +} + +TEST_F(NewParquetReaderTest, ReportsPersistentFileMetaCacheProfile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(0, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_persistent_file_meta_cache_first"); + auto first_reader = + create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(first_reader->init(&state).ok()); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterHitDiskCache")->value(), 0); + EXPECT_EQ(first_profile.get_counter("FileFooterMissDiskCache")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_persistent_file_meta_cache_second"); + auto second_reader = + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterMissDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterReadDiskCacheTime"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterWriteDiskCacheTime"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterMissDiskCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterWriteDiskCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, InvalidPersistentFileMetaCachePayloadFallsBackToFile) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + auto persistent_cache = std::make_unique(); + TestPersistentFileMetaCache* persistent_cache_ptr = persistent_cache.get(); + FileMetaCache file_meta_cache(0, std::move(persistent_cache)); + + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + ASSERT_TRUE(persistent_cache_ptr + ->write(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key, 123, + file_size, "not a serialized parquet footer") + .ok()); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("new_parquet_reader_invalid_persistent_file_meta_cache_payload"); + auto reader = create_reader(0, -1, &profile, false, nullptr, std::nullopt, &file_meta_cache); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + EXPECT_EQ(persistent_cache_ptr->remove_count(), 1); + EXPECT_TRUE( + persistent_cache_ptr->contains(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key)); + EXPECT_NE(persistent_cache_ptr->payload(FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key), + "not a serialized parquet footer"); +} + +TEST_F(NewParquetReaderTest, PersistentFileMetaCacheCanSkipMemoryCache) { + const bool old_enable_external_file_meta_disk_cache = + config::enable_external_file_meta_disk_cache; + const int64_t old_external_file_meta_disk_cache_max_entry_bytes = + config::external_file_meta_disk_cache_max_entry_bytes; + Defer defer {[&] { + config::enable_external_file_meta_disk_cache = old_enable_external_file_meta_disk_cache; + config::external_file_meta_disk_cache_max_entry_bytes = + old_external_file_meta_disk_cache_max_entry_bytes; + }}; + config::enable_external_file_meta_disk_cache = true; + config::external_file_meta_disk_cache_max_entry_bytes = 64 * 1024 * 1024; + + FileMetaCache file_meta_cache(1024 * 1024, std::make_unique()); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("new_parquet_reader_skip_memory_file_meta_cache_first"); + auto first_reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, + &file_meta_cache, false); + ASSERT_TRUE(first_reader->init(&state).ok()); + const auto file_size = static_cast(std::filesystem::file_size(_file_path)); + const std::string file_meta_cache_key = FileMetaCache::get_key(_file_path, 123, file_size); + const std::string memory_cache_key = FileMetaCache::get_memory_cache_key( + FileMetaCacheFormat::PARQUET_V2, file_meta_cache_key); + ObjLRUCache::CacheHandle handle; + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); + ASSERT_NE(first_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(first_profile.get_counter("FileFooterWriteDiskCache"), nullptr); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterWriteDiskCache")->value(), 1); + + RuntimeProfile second_profile("new_parquet_reader_skip_memory_file_meta_cache_second"); + auto second_reader = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, + &file_meta_cache, false); + ASSERT_TRUE(second_reader->init(&state).ok()); + ASSERT_NE(second_profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitMemoryCache"), nullptr); + ASSERT_NE(second_profile.get_counter("FileFooterHitDiskCache"), nullptr); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitMemoryCache")->value(), 0); + EXPECT_EQ(second_profile.get_counter("FileFooterHitDiskCache")->value(), 1); + EXPECT_FALSE(file_meta_cache.lookup(memory_cache_key, &handle)); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. @@ -1423,7 +1666,8 @@ TEST_F(NewParquetReaderTest, ReadMultipleRowGroups) { TEST_F(NewParquetReaderTest, RewriteSameLocalPathDoesNotReuseUnknownMtimePageCache) { RuntimeProfile first_profile("new_parquet_reader_first_unknown_mtime"); { - auto reader = create_reader(0, -1, &first_profile); + auto reader = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, nullptr, + true, 0); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -1450,7 +1694,8 @@ TEST_F(NewParquetReaderTest, RewriteSameLocalPathDoesNotReuseUnknownMtimePageCac // from the previous physical file, even when the query option enables parquet file page cache. write_int_pair_parquet_file(_file_path); RuntimeProfile second_profile("new_parquet_reader_second_unknown_mtime"); - auto reader = create_reader(0, -1, &second_profile); + auto reader = + create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, nullptr, true, 0); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 8a10fc06a41db3..1eef2dfa6c263f 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -62,6 +62,8 @@ #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" #include "gen_cpp/PlanNodes_types.h" +#include "io/fs/file_meta_cache.h" +#include "io/fs/local_file_system.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" #include "runtime/runtime_profile.h" @@ -2018,6 +2020,71 @@ TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { std::filesystem::remove_all(test_dir); } +TEST(IcebergV2ReaderTest, IcebergDeleteReaderInheritsFileMetaMemoryCachePolicy) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_delete_file_meta_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache delete_file_cache(1); + + auto delete_footer_is_cached = [&](bool enable_file_meta_memory_cache) { + FileMetaCache file_meta_cache(1024); + doris::format::iceberg::IcebergTableReader reader; + EXPECT_TRUE( + reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + .file_meta_cache = &file_meta_cache, + .enable_file_meta_memory_cache = enable_file_meta_memory_cache, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &delete_file_cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + EXPECT_TRUE(reader.prepare_split(split_options).ok()); + + io::FileReaderSPtr delete_file_reader; + EXPECT_TRUE(io::global_local_filesystem() + ->open_file(delete_file_path, &delete_file_reader) + .ok()); + io::FileDescription file_description; + file_description.mtime = 0; + file_description.file_size = -1; + const std::string meta_key = FileMetaCache::get_key(delete_file_reader, file_description); + const std::string memory_key = + FileMetaCache::get_memory_cache_key(FileMetaCacheFormat::PARQUET_V2, meta_key); + ObjLRUCache::CacheHandle handle; + const bool cached = file_meta_cache.lookup(memory_key, &handle); + EXPECT_TRUE(reader.close().ok()); + return cached; + }; + + EXPECT_TRUE(delete_footer_is_cached(true)); + EXPECT_FALSE(delete_footer_is_cached(false)); + std::filesystem::remove_all(test_dir); +} + TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesNullablePositionDeleteFileWithoutNulls) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_nullable_position_delete_file_test";