From ca66ec6c4d4d788b1e6cd8b13e6f91fd734b613a Mon Sep 17 00:00:00 2001 From: csun5285 Date: Sun, 12 Jul 2026 12:44:53 +0800 Subject: [PATCH 1/3] [refactor](storage) extract RowKeyEncoder: dedup key encoding across both segment writers SegmentWriter and VerticalSegmentWriter carry two verbatim copies of the row-key encoding logic (_full_encode_keys x2, _encode_seq_column, _encode_rowid, _encode_keys, plus six coder members each). Extract it into a shared RowKeyEncoder (be/src/storage/key/row_key_encoder.{h,cpp}) with the two views made explicit: - full_encode() / encode_short_keys(): the sort-key view (cluster key columns for mow tables with cluster keys, schema key columns otherwise) - full_encode_primary_keys(): the primary-key view (always the schema key columns; the primary key index is built on these even when the segment sorts by cluster keys) - append_seq_suffix() / append_rowid_suffix(): the two key suffixes Both writers now hold a RowKeyEncoder member and _generate_primary_key_index loses its coder parameter. BlockAggregator's calls into the writer's encode helpers in partial_update_info.cpp are redirected onto the writer's _key_encoder (decoupling BlockAggregator from the writer entirely is left to a follow-up). Equivalence notes for review: - SegmentWriter's _full_encode_keys had a null_first parameter that no caller ever set to false; the encoder always uses the null-first marker. - The constructor DCHECK(num_sort_key_columns >= num_short_key_columns) is dropped; both counts now derive from TabletSchema in one place. - SegmentWriter::set_mow_context had zero callers and is removed. - _maybe_invalid_row_cache intentionally stays in the writers; moving the row-cache invalidation is out of scope here. Tests: row_key_encoder_test.cpp with 30 cases - 13 behavior cases (marker layout, null-first ordering, sort-key vs primary-key views, short-key truncation and collision, seq/rowid suffixes) plus a golden byte matrix that pins the encoded bytes of every legal key-column type (TINYINT, SMALLINT, INT, BIGINT, LARGEINT, BOOL, DATEV2, DATETIMEV2, DECIMAL32/64/ 128I/256, IPV4, IPV6, CHAR, VARCHAR, STRING) in all three views as literal hex strings, so any coder change or CPU-architecture drift fails loudly. Every existing MoW / partial-update / pk-index UT now executes through RowKeyEncoder. Co-Authored-By: Claude Fable 5 --- be/src/storage/key/row_key_encoder.cpp | 137 ++++ be/src/storage/key/row_key_encoder.h | 88 +++ be/src/storage/partial_update_info.cpp | 11 +- be/src/storage/segment/segment_writer.cpp | 153 +---- be/src/storage/segment/segment_writer.h | 30 +- .../segment/vertical_segment_writer.cpp | 168 +---- .../storage/segment/vertical_segment_writer.h | 26 +- be/test/storage/key/row_key_encoder_test.cpp | 637 ++++++++++++++++++ be/test/storage/segment/test_segment_writer.h | 2 +- 9 files changed, 919 insertions(+), 333 deletions(-) create mode 100644 be/src/storage/key/row_key_encoder.cpp create mode 100644 be/src/storage/key/row_key_encoder.h create mode 100644 be/test/storage/key/row_key_encoder_test.cpp diff --git a/be/src/storage/key/row_key_encoder.cpp b/be/src/storage/key/row_key_encoder.cpp new file mode 100644 index 00000000000000..501ea75cab03af --- /dev/null +++ b/be/src/storage/key/row_key_encoder.cpp @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "storage/key/row_key_encoder.h" + +#include + +#include "common/cast_set.h" +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/consts.h" +#include "common/logging.h" +#include "storage/iterator/olap_data_convertor.h" +#include "storage/key_coder.h" +#include "storage/tablet/tablet_schema.h" + +namespace doris { + +RowKeyEncoder::RowKeyEncoder(const TabletSchema& schema, bool mow) { + for (size_t cid = 0; cid < schema.num_key_columns(); ++cid) { + _primary_key_coders.push_back(get_key_coder(schema.column(cid).type())); + } + _num_short_key_columns = schema.num_short_key_columns(); + // encode the sequence id into the primary key index + if (schema.has_sequence_col()) { + const auto& column = schema.column(schema.sequence_col_idx()); + _seq_coder = get_key_coder(column.type()); + _seq_col_length = column.length(); + } + if (mow && !schema.cluster_key_uids().empty()) { + _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); + for (auto uid : schema.cluster_key_uids()) { + _add_sort_key_column(schema.column_by_uid(uid)); + } + } else { + for (size_t cid = 0; cid < schema.num_key_columns(); ++cid) { + _add_sort_key_column(schema.column(cid)); + } + } +} + +void RowKeyEncoder::_add_sort_key_column(const TabletColumn& column) { + _sort_key_coders.push_back(get_key_coder(column.type())); + _sort_key_index_size.push_back(cast_set(column.index_length())); +} + +std::string RowKeyEncoder::full_encode(const std::vector& key_columns, + size_t pos) const { + assert(_sort_key_index_size.size() == _sort_key_coders.size()); + assert(key_columns.size() == _sort_key_coders.size()); + return _full_encode(_sort_key_coders, key_columns, pos); +} + +std::string RowKeyEncoder::full_encode_primary_keys( + const std::vector& key_columns, size_t pos) const { + return _full_encode(_primary_key_coders, key_columns, pos); +} + +namespace { +// Shared row-key encoding base: for each key column, write a null marker for +// a null value, otherwise a normal marker followed by whatever `encode_field` +// appends. `encode_field(cid, field, out)` is the only thing that differs between +// the full key encode and the short-key prefix encode. +template +std::string encode_key_columns(const std::vector& key_columns, size_t pos, + EncodeField&& encode_field) { + std::string encoded_keys; + size_t cid = 0; + for (const auto& column : key_columns) { + const auto* field = column->get_data_at(pos); + if (UNLIKELY(!field)) { + encoded_keys.push_back(KeyConsts::KEY_NULL_FIRST_MARKER); + ++cid; + continue; + } + encoded_keys.push_back(KeyConsts::KEY_NORMAL_MARKER); + encode_field(cid, field, &encoded_keys); + ++cid; + } + return encoded_keys; +} +} // namespace + +std::string RowKeyEncoder::_full_encode(const std::vector& key_coders, + const std::vector& key_columns, + size_t pos) { + assert(key_columns.size() == key_coders.size()); + return encode_key_columns(key_columns, pos, + [&](size_t cid, const void* field, std::string* out) { + DCHECK(key_coders[cid] != nullptr); + key_coders[cid]->full_encode_ascending(field, out); + }); +} + +std::string RowKeyEncoder::encode_short_keys( + const std::vector& key_columns, size_t pos) const { + assert(key_columns.size() == _num_short_key_columns); + assert(key_columns.size() <= _sort_key_coders.size()); + return encode_key_columns( + key_columns, pos, [&](size_t cid, const void* field, std::string* out) { + _sort_key_coders[cid]->encode_ascending(field, _sort_key_index_size[cid], out); + }); +} + +void RowKeyEncoder::append_seq_suffix(std::string* encoded_keys, + const IOlapColumnDataAccessor* seq_column, size_t pos) const { + const auto* field = seq_column->get_data_at(pos); + // So the primary key index can still use it, encode a null seq column as + // the smallest value of its length. + if (UNLIKELY(!field)) { + encoded_keys->push_back(KeyConsts::KEY_NULL_FIRST_MARKER); + encoded_keys->append(_seq_col_length, KeyConsts::KEY_MINIMAL_MARKER); + return; + } + encoded_keys->push_back(KeyConsts::KEY_NORMAL_MARKER); + _seq_coder->full_encode_ascending(field, encoded_keys); +} + +void RowKeyEncoder::append_rowid_suffix(std::string* encoded_keys, uint32_t rowid) const { + encoded_keys->push_back(KeyConsts::KEY_NORMAL_MARKER); + _rowid_coder->full_encode_ascending(&rowid, encoded_keys); +} + +} // namespace doris diff --git a/be/src/storage/key/row_key_encoder.h b/be/src/storage/key/row_key_encoder.h new file mode 100644 index 00000000000000..c2e8a935156a6d --- /dev/null +++ b/be/src/storage/key/row_key_encoder.h @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris { + +class IOlapColumnDataAccessor; +class KeyCoder; +class TabletColumn; +class TabletSchema; + +// Encodes rows into the sortable binary key format shared by the short key +// index and the primary key index. Each column is encoded as a one byte +// marker (KeyConsts) followed by the KeyCoder encoded value, a null column +// is encoded as KEY_NULL_FIRST_MARKER without value bytes. +class RowKeyEncoder { +public: + RowKeyEncoder(const TabletSchema& schema, bool mow); + + // Encode the sort key columns at `pos` with full length. + std::string full_encode(const std::vector& key_columns, + size_t pos) const; + + // Encode the primary key columns at `pos` with full length, producing the + // key stored in and probed against the primary key index. + std::string full_encode_primary_keys(const std::vector& key_columns, + size_t pos) const; + + // Encode the short key columns at `pos`, each column truncated to its + // index length. + std::string encode_short_keys(const std::vector& key_columns, + size_t pos) const; + + // Append the encoded sequence column at `pos` to `encoded_keys`. A null + // sequence value is encoded as the minimal value of the column length so + // that it sorts first in the primary key index. + void append_seq_suffix(std::string* encoded_keys, const IOlapColumnDataAccessor* seq_column, + size_t pos) const; + + // Append the encoded row id to `encoded_keys`, only used by mow tables + // with cluster keys. + void append_rowid_suffix(std::string* encoded_keys, uint32_t rowid) const; + + size_t num_sort_key_columns() const { return _sort_key_coders.size(); } + +private: + static std::string _full_encode(const std::vector& key_coders, + const std::vector& key_columns, + size_t pos); + + void _add_sort_key_column(const TabletColumn& column); + + // The sort-key view: whatever the segment sorts by. Cluster key columns + // for mow tables with cluster keys, primary key columns otherwise. Used by + // full_encode() and encode_short_keys(). + std::vector _sort_key_coders; + std::vector _sort_key_index_size; + // The primary-key view: always the primary key columns. The primary key + // index is built on these even when the segment sorts by cluster keys; + // used by full_encode_primary_keys(). + std::vector _primary_key_coders; + const KeyCoder* _seq_coder = nullptr; + const KeyCoder* _rowid_coder = nullptr; + size_t _seq_col_length = 0; + size_t _num_short_key_columns = 0; +}; + +} // namespace doris diff --git a/be/src/storage/partial_update_info.cpp b/be/src/storage/partial_update_info.cpp index 4adbd834251a42..b41df7384722d7 100644 --- a/be/src/storage/partial_update_info.cpp +++ b/be/src/storage/partial_update_info.cpp @@ -29,6 +29,7 @@ #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/value/bitmap_value.h" #include "storage/iterator/olap_data_convertor.h" +#include "storage/key/row_key_encoder.h" #include "storage/olap_common.h" #include "storage/rowset/rowset.h" #include "storage/rowset/rowset_writer_context.h" @@ -915,7 +916,7 @@ Status BlockAggregator::aggregate_rows( // Discard all the rows whose seq value is smaller than previous_encoded_seq_value. if (row_has_sequence_col) { std::string seq_val {}; - _writer._encode_seq_column(seq_column, pos, &seq_val); + _writer._key_encoder.append_seq_suffix(&seq_val, seq_column, pos); if (Slice {seq_val}.compare(Slice {previous_encoded_seq_value}) < 0) { continue; } @@ -932,7 +933,7 @@ Status BlockAggregator::aggregate_rows( if (row_has_sequence_col) { std::string seq_val {}; // for rows that don't specify seqeunce col, seq_val will be encoded to minial value - _writer._encode_seq_column(seq_column, pos, &seq_val); + _writer._key_encoder.append_seq_suffix(&seq_val, seq_column, pos); cur_seq_val = std::move(seq_val); } else { cur_seq_val.clear(); @@ -950,7 +951,7 @@ Status BlockAggregator::aggregate_rows( append_or_merge_row(output_block, block, rid, skip_bitmap, have_delete_sign); } else { std::string seq_val {}; - _writer._encode_seq_column(seq_column, rid, &seq_val); + _writer._key_encoder.append_seq_suffix(&seq_val, seq_column, rid); if (Slice {seq_val}.compare(Slice {cur_seq_val}) >= 0) { append_or_merge_row(output_block, block, rid, skip_bitmap, have_delete_sign); cur_seq_val = std::move(seq_val); @@ -981,7 +982,7 @@ Status BlockAggregator::aggregate_for_sequence_column( int same_key_rows {0}; std::string previous_key {}; for (int block_pos {0}; block_pos < num_rows; block_pos++) { - std::string key = _writer._full_encode_keys(key_columns, block_pos); + std::string key = _writer._key_encoder.full_encode(key_columns, block_pos); if (block_pos > 0 && previous_key == key) { same_key_rows++; } else { @@ -1061,7 +1062,7 @@ Status BlockAggregator::aggregate_for_insert_after_delete( for (size_t block_pos {0}; block_pos < num_rows; block_pos++) { size_t delta_pos = block_pos; auto& skip_bitmap = skip_bitmaps->at(block_pos); - std::string key = _writer._full_encode_keys(key_columns, delta_pos); + std::string key = _writer._key_encoder.full_encode(key_columns, delta_pos); bool have_delete_sign = (!skip_bitmap.contains(delete_sign_col_unique_id) && delete_signs[block_pos] != 0); if (delta_pos > 0 && previous_key == key) { diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index 6f4ecef1140d67..970e5589e7f6bc 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -81,7 +81,6 @@ namespace doris { namespace segment_v2 { using namespace ErrorCode; -using namespace KeyConsts; const char* k_segment_magic = "D0R1"; const uint32_t k_segment_magic_length = 4; @@ -102,44 +101,10 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, _file_writer(file_writer), _index_file_writer(index_file_writer), _mem_tracker(std::make_unique(segment_mem_tracker_name(segment_id))), + _key_encoder(*_tablet_schema, _is_mow()), _mow_context(std::move(opts.mow_ctx)) { CHECK_NOTNULL(file_writer); - _num_sort_key_columns = _tablet_schema->num_key_columns(); _num_short_key_columns = _tablet_schema->num_short_key_columns(); - if (!_is_mow_with_cluster_key()) { - DCHECK(_num_sort_key_columns >= _num_short_key_columns) - << ", table_id=" << _tablet_schema->table_id() - << ", num_key_columns=" << _num_sort_key_columns - << ", num_short_key_columns=" << _num_short_key_columns - << ", cluster_key_columns=" << _tablet_schema->cluster_key_uids().size(); - } - for (size_t cid = 0; cid < _num_sort_key_columns; ++cid) { - const auto& column = _tablet_schema->column(cid); - _key_coders.push_back(get_key_coder(column.type())); - _key_index_size.push_back(cast_set(column.index_length())); - } - if (_is_mow()) { - // encode the sequence id into the primary key index - if (_tablet_schema->has_sequence_col()) { - const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); - _seq_coder = get_key_coder(column.type()); - } - // encode the rowid into the primary key index - if (_is_mow_with_cluster_key()) { - _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); - // primary keys - _primary_key_coders.swap(_key_coders); - // cluster keys - _key_coders.clear(); - _key_index_size.clear(); - _num_sort_key_columns = _tablet_schema->cluster_key_uids().size(); - for (auto cid : _tablet_schema->cluster_key_uids()) { - const auto& column = _tablet_schema->column_by_uid(cid); - _key_coders.push_back(get_key_coder(column.type())); - _key_index_size.push_back(cast_set(column.index_length())); - } - } - } } SegmentWriter::~SegmentWriter() { @@ -552,7 +517,7 @@ Status SegmentWriter::append_block_with_partial_content(const Block* block, size if (!converted_result.first.ok()) { return converted_result.first; } - if (cid < _num_sort_key_columns) { + if (cid < _key_encoder.num_sort_key_columns()) { key_columns.push_back(converted_result.second); } else if (_tablet_schema->has_sequence_col() && cid == _tablet_schema->sequence_col_idx()) { @@ -587,10 +552,10 @@ Status SegmentWriter::append_block_with_partial_content(const Block* block, size // here row_pos = 2, num_rows = 4. size_t delta_pos = block_pos - row_pos; size_t segment_pos = segment_start_pos + delta_pos; - std::string key = _full_encode_keys(key_columns, delta_pos); + std::string key = _key_encoder.full_encode(key_columns, delta_pos); _maybe_invalid_row_cache(key); if (have_input_seq_column) { - _encode_seq_column(seq_column, delta_pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, delta_pos); } // If the table have sequence column, and the include-cids don't contain the sequence // column, we need to update the primary key index builder at the end of this method. @@ -605,8 +570,8 @@ Status SegmentWriter::append_block_with_partial_content(const Block* block, size auto not_found_cb = [&]() { return _opts.rowset_ctx->partial_update_info->handle_new_key( *_tablet_schema, [&]() -> std::string { - return block->dump_one_line(block_pos, - cast_set(_num_sort_key_columns)); + return block->dump_one_line( + block_pos, cast_set(_key_encoder.num_sort_key_columns())); }); }; auto update_read_plan = [&](const RowLocation& loc) { @@ -668,8 +633,7 @@ Status SegmentWriter::append_block_with_partial_content(const Block* block, size "index builder num rows: {}", _num_rows_written, row_pos, _primary_key_index_builder->num_rows()); } - RETURN_IF_ERROR( - _generate_primary_key_index(_key_coders, key_columns, seq_column, num_rows, false)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, num_rows, false)); } _num_rows_written += num_rows; @@ -772,8 +736,7 @@ Status SegmentWriter::build_key_index(std::vector& key if (_is_mow_with_cluster_key()) { // For CLUSTER BY tables: // 1) generate primary key index (unique keys) - RETURN_IF_ERROR(_generate_primary_key_index(_primary_key_coders, key_columns, seq_column, - num_rows, true)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, num_rows, true)); // 2) generate short key index (cluster keys) key_columns.clear(); for (const auto& cid : _tablet_schema->cluster_key_uids()) { @@ -803,7 +766,7 @@ Status SegmentWriter::build_key_index(std::vector& key return _generate_short_key_index(key_columns, num_rows, short_key_pos); } if (_is_mow()) { - return _generate_primary_key_index(_key_coders, key_columns, seq_column, num_rows, false); + return _generate_primary_key_index(key_columns, seq_column, num_rows, false); } return _generate_short_key_index(key_columns, num_rows, short_key_pos); } @@ -820,81 +783,6 @@ int64_t SegmentWriter::max_row_to_add(size_t row_avg_size_in_bytes) { return std::min(size_rows, count_rows); } -std::string SegmentWriter::_full_encode_keys( - const std::vector& key_columns, size_t pos, bool null_first) { - assert(_key_index_size.size() == _num_sort_key_columns); - assert(key_columns.size() == _num_sort_key_columns && - _key_coders.size() == _num_sort_key_columns); - return _full_encode_keys(_key_coders, key_columns, pos, null_first); -} - -std::string SegmentWriter::_full_encode_keys( - const std::vector& key_coders, - const std::vector& key_columns, size_t pos, bool null_first) { - assert(key_columns.size() == key_coders.size()); - - std::string encoded_keys; - size_t cid = 0; - for (const auto& column : key_columns) { - auto field = column->get_data_at(pos); - if (UNLIKELY(!field)) { - if (null_first) { - encoded_keys.push_back(KEY_NULL_FIRST_MARKER); - } else { - encoded_keys.push_back(KEY_NORMAL_MARKER); - } - ++cid; - continue; - } - encoded_keys.push_back(KEY_NORMAL_MARKER); - DCHECK(key_coders[cid] != nullptr); - key_coders[cid]->full_encode_ascending(field, &encoded_keys); - ++cid; - } - return encoded_keys; -} - -void SegmentWriter::_encode_seq_column(const IOlapColumnDataAccessor* seq_column, size_t pos, - std::string* encoded_keys) { - auto field = seq_column->get_data_at(pos); - // To facilitate the use of the primary key index, encode the seq column - // to the minimum value of the corresponding length when the seq column - // is null - if (UNLIKELY(!field)) { - encoded_keys->push_back(KEY_NULL_FIRST_MARKER); - size_t seq_col_length = _tablet_schema->column(_tablet_schema->sequence_col_idx()).length(); - encoded_keys->append(seq_col_length, KEY_MINIMAL_MARKER); - return; - } - encoded_keys->push_back(KEY_NORMAL_MARKER); - _seq_coder->full_encode_ascending(field, encoded_keys); -} - -void SegmentWriter::_encode_rowid(const uint32_t rowid, std::string* encoded_keys) { - encoded_keys->push_back(KEY_NORMAL_MARKER); - _rowid_coder->full_encode_ascending(&rowid, encoded_keys); -} - -std::string SegmentWriter::_encode_keys(const std::vector& key_columns, - size_t pos) { - assert(key_columns.size() == _num_short_key_columns); - - std::string encoded_keys; - size_t cid = 0; - for (const auto& column : key_columns) { - auto field = column->get_data_at(pos); - if (UNLIKELY(!field)) { - encoded_keys.push_back(KEY_NULL_FIRST_MARKER); - ++cid; - continue; - } - encoded_keys.push_back(KEY_NORMAL_MARKER); - _key_coders[cid]->encode_ascending(field, _key_index_size[cid], &encoded_keys); - ++cid; - } - return encoded_keys; -} - // TODO(lingbin): Currently this function does not include the size of various indexes, // We should make this more precise. // NOTE: This function will be called when any row of data is added, so we need to @@ -1198,22 +1086,16 @@ void SegmentWriter::set_max_key(const Slice& key) { _max_key.append(key.get_data(), key.get_size()); } -void SegmentWriter::set_mow_context(std::shared_ptr mow_context) { - _mow_context = mow_context; -} - Status SegmentWriter::_generate_primary_key_index( - const std::vector& primary_key_coders, const std::vector& primary_key_columns, IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort) { if (!need_sort) { // mow table without cluster key std::string last_key; for (size_t pos = 0; pos < num_rows; pos++) { - // use _key_coders - std::string key = _full_encode_keys(primary_key_columns, pos); + std::string key = _key_encoder.full_encode(primary_key_columns, pos); _maybe_invalid_row_cache(key); if (_tablet_schema->has_sequence_col()) { - _encode_seq_column(seq_column, pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, pos); } DCHECK(key.compare(last_key) > 0) << "found duplicate key or key is not sorted! current key: " << key @@ -1224,12 +1106,12 @@ Status SegmentWriter::_generate_primary_key_index( } else { // mow table with cluster key // generate primary keys in memory for (uint32_t pos = 0; pos < num_rows; pos++) { - std::string key = _full_encode_keys(primary_key_coders, primary_key_columns, pos); + std::string key = _key_encoder.full_encode_primary_keys(primary_key_columns, pos); _maybe_invalid_row_cache(key); if (_tablet_schema->has_sequence_col()) { - _encode_seq_column(seq_column, pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, pos); } - _encode_rowid(pos + _num_rows_written, &key); + _key_encoder.append_rowid_suffix(&key, pos + _num_rows_written); _primary_keys_size += key.size(); _primary_keys.emplace_back(std::move(key)); } @@ -1240,9 +1122,8 @@ Status SegmentWriter::_generate_primary_key_index( Status SegmentWriter::_generate_short_key_index(std::vector& key_columns, size_t num_rows, const std::vector& short_key_pos) { - // use _key_coders - set_min_key(_full_encode_keys(key_columns, 0)); - set_max_key(_full_encode_keys(key_columns, num_rows - 1)); + set_min_key(_key_encoder.full_encode(key_columns, 0)); + set_max_key(_key_encoder.full_encode(key_columns, num_rows - 1)); DCHECK(Slice(_max_key.data(), _max_key.size()) .compare(Slice(_min_key.data(), _min_key.size())) >= 0) << "key is not sorted! min key: " << _min_key << ", max key: " << _max_key; @@ -1250,7 +1131,7 @@ Status SegmentWriter::_generate_short_key_index(std::vector= 0) << "key is not sorted! current key: " << key << ", last key: " << last_key; RETURN_IF_ERROR(_short_key_index_builder->add_item(key)); diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h index 4f6ce5c866159b..a3f7915dc68c77 100644 --- a/be/src/storage/segment/segment_writer.h +++ b/be/src/storage/segment/segment_writer.h @@ -31,6 +31,7 @@ #include "common/status.h" // Status #include "storage/index/index_file_writer.h" +#include "storage/key/row_key_encoder.h" #include "storage/olap_define.h" #include "storage/segment/column_writer.h" #include "storage/segment/segment_index_file_cache_loader.h" @@ -136,8 +137,6 @@ class SegmentWriter { void clear(); - void set_mow_context(std::shared_ptr mow_context); - Status close_inverted_index(int64_t* inverted_index_file_size) { // no inverted index if (_index_file_writer == nullptr) { @@ -169,25 +168,11 @@ class SegmentWriter { Status _write_footer(); Status _write_raw_data(const std::vector& slices); void _maybe_invalid_row_cache(const std::string& key); - std::string _encode_keys(const std::vector& key_columns, size_t pos); - // used for unique-key with merge on write and segment min_max key - std::string _full_encode_keys(const std::vector& key_columns, - size_t pos, bool null_first = true); - - static std::string _full_encode_keys(const std::vector& key_coders, - const std::vector& key_columns, - size_t pos, bool null_first = true); - - // used for unique-key with merge on write - void _encode_seq_column(const IOlapColumnDataAccessor* seq_column, size_t pos, - std::string* encoded_keys); - void _encode_rowid(const uint32_t rowid, std::string* encoded_keys); void set_min_max_key(const Slice& key); void set_min_key(const Slice& key); void set_max_key(const Slice& key); void _serialize_block_to_row_column(Block& block); Status _generate_primary_key_index( - const std::vector& primary_key_coders, const std::vector& primary_key_columns, IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort); Status _generate_short_key_index(std::vector& key_columns, @@ -217,9 +202,6 @@ class SegmentWriter { SegmentFooterPB _footer; SegmentIndexFileCacheInfo _index_file_cache_info; - // for mow tables with cluster key, the sort key is the cluster keys not unique keys - // for other tables, the sort key is the keys - size_t _num_sort_key_columns; size_t _num_short_key_columns; std::unique_ptr _short_key_index_builder; @@ -229,13 +211,9 @@ class SegmentWriter { std::unique_ptr _olap_data_convertor; // used for building short key index or primary key index during vectorized write. - // for mow table with cluster keys, this is cluster keys - std::vector _key_coders; - // for mow table with cluster keys, this is primary keys - std::vector _primary_key_coders; - const KeyCoder* _seq_coder = nullptr; - const KeyCoder* _rowid_coder = nullptr; - std::vector _key_index_size; + // NOTE: must stay declared after _tablet_schema and _opts, the constructor + // init list reads both through _is_mow(). + RowKeyEncoder _key_encoder; size_t _short_key_row_pos = 0; std::vector _column_ids; diff --git a/be/src/storage/segment/vertical_segment_writer.cpp b/be/src/storage/segment/vertical_segment_writer.cpp index f5b60ebadb756c..ea9bae3a9266f2 100644 --- a/be/src/storage/segment/vertical_segment_writer.cpp +++ b/be/src/storage/segment/vertical_segment_writer.cpp @@ -86,7 +86,6 @@ namespace doris::segment_v2 { using namespace ErrorCode; -using namespace KeyConsts; static constexpr const char* k_segment_magic = "D0R1"; static constexpr uint32_t k_segment_magic_length = 4; @@ -117,45 +116,11 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 _index_file_writer(index_file_writer), _mem_tracker(std::make_unique( vertical_segment_writer_mem_tracker_name(segment_id))), + _key_encoder(*_tablet_schema, _is_mow()), _mow_context(std::move(opts.mow_ctx)), _block_aggregator(*this) { CHECK_NOTNULL(file_writer); - _num_sort_key_columns = _tablet_schema->num_key_columns(); _num_short_key_columns = _tablet_schema->num_short_key_columns(); - if (!_is_mow_with_cluster_key()) { - DCHECK(_num_sort_key_columns >= _num_short_key_columns) - << ", table_id=" << _tablet_schema->table_id() - << ", num_key_columns=" << _num_sort_key_columns - << ", num_short_key_columns=" << _num_short_key_columns - << ", cluster_key_columns=" << _tablet_schema->cluster_key_uids().size(); - } - for (size_t cid = 0; cid < _num_sort_key_columns; ++cid) { - const auto& column = _tablet_schema->column(cid); - _key_coders.push_back(get_key_coder(column.type())); - _key_index_size.push_back(cast_set(column.index_length())); - } - // encode the sequence id into the primary key index - if (_is_mow()) { - if (_tablet_schema->has_sequence_col()) { - const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); - _seq_coder = get_key_coder(column.type()); - } - // encode the rowid into the primary key index - if (_is_mow_with_cluster_key()) { - _rowid_coder = get_key_coder(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT); - // primary keys - _primary_key_coders.swap(_key_coders); - // cluster keys - _key_coders.clear(); - _key_index_size.clear(); - _num_sort_key_columns = _tablet_schema->cluster_key_uids().size(); - for (auto cid : _tablet_schema->cluster_key_uids()) { - const auto& column = _tablet_schema->column_by_uid(cid); - _key_coders.push_back(get_key_coder(column.type())); - _key_index_size.push_back(cast_set(column.index_length())); - } - } - } } VerticalSegmentWriter::~VerticalSegmentWriter() { @@ -578,7 +543,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da if (!status.ok()) { return status; } - if (cid < _num_sort_key_columns) { + if (cid < _key_encoder.num_sort_key_columns()) { key_columns.push_back(column); } else if (_tablet_schema->has_sequence_col() && cid == _tablet_schema->sequence_col_idx()) { @@ -589,9 +554,9 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da data.num_rows)); RETURN_IF_ERROR(_finalize_column_writer_and_update_meta(cid)); // Don't clear source content for key columns and sequence column here, - // as they will be used later in _full_encode_keys() and _generate_primary_key_index(). + // as they will be used later for key encoding and _generate_primary_key_index(). // They will be cleared at the end of this method. - bool is_key_column = (cid < _num_sort_key_columns); + bool is_key_column = (cid < _key_encoder.num_sort_key_columns()); bool is_seq_column = (_tablet_schema->has_sequence_col() && cid == _tablet_schema->sequence_col_idx() && have_input_seq_column); if (!is_key_column && !is_seq_column) { @@ -624,10 +589,10 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da // here row_pos = 2, num_rows = 4. size_t delta_pos = block_pos - data.row_pos; size_t segment_pos = segment_start_pos + delta_pos; - std::string key = _full_encode_keys(key_columns, delta_pos); + std::string key = _key_encoder.full_encode(key_columns, delta_pos); _maybe_invalid_row_cache(key); if (have_input_seq_column) { - _encode_seq_column(seq_column, delta_pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, delta_pos); } // If the table have sequence column, and the include-cids don't contain the sequence // column, we need to update the primary key index builder at the end of this method. @@ -642,8 +607,8 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da auto not_found_cb = [&]() { return _opts.rowset_ctx->partial_update_info->handle_new_key( *_tablet_schema, [&]() -> std::string { - return data.block->dump_one_line(block_pos, - cast_set(_num_sort_key_columns)); + return data.block->dump_one_line( + block_pos, cast_set(_key_encoder.num_sort_key_columns())); }); }; auto update_read_plan = [&](const RowLocation& loc) { @@ -717,8 +682,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da "index builder num rows: {}", _num_rows_written, data.row_pos, _primary_key_index_builder->num_rows()); } - RETURN_IF_ERROR(_generate_primary_key_index(_key_coders, key_columns, seq_column, - data.num_rows, false)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, data.num_rows, false)); } _num_rows_written += data.num_rows; @@ -899,8 +863,7 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content(RowsIn } // 9. build primary key index - RETURN_IF_ERROR(_generate_primary_key_index(_key_coders, key_columns, seq_column, data.num_rows, - false)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, data.num_rows, false)); _num_rows_written += data.num_rows; DCHECK_EQ(_primary_key_index_builder->num_rows(), _num_rows_written) @@ -935,7 +898,7 @@ Status VerticalSegmentWriter::_generate_encoded_default_seq_value(const TabletSc return status; } // include marker - _encode_seq_column(column, 0, encoded_value); + _key_encoder.append_seq_suffix(encoded_value, column, 0); return Status::OK(); } @@ -960,12 +923,12 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( size_t segment_pos = segment_start_pos + delta_pos; auto& skip_bitmap = skip_bitmaps->at(block_pos); - std::string key = _full_encode_keys(key_columns, delta_pos); + std::string key = _key_encoder.full_encode(key_columns, delta_pos); _maybe_invalid_row_cache(key); bool row_has_sequence_col = (schema_has_sequence_col && !skip_bitmap.contains(seq_col_unique_id)); if (row_has_sequence_col) { - _encode_seq_column(seq_column, delta_pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, delta_pos); } // mark key with delete sign as deleted. @@ -976,8 +939,8 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( return _opts.rowset_ctx->partial_update_info->handle_new_key( *_tablet_schema, [&]() -> std::string { - return data.block->dump_one_line(block_pos, - cast_set(_num_sort_key_columns)); + return data.block->dump_one_line( + block_pos, cast_set(_key_encoder.num_sort_key_columns())); }, &skip_bitmap); }; @@ -1141,8 +1104,7 @@ Status VerticalSegmentWriter::_generate_key_index( } if (_is_mow_with_cluster_key()) { // 1. generate primary key index - RETURN_IF_ERROR(_generate_primary_key_index(_primary_key_coders, key_columns, seq_column, - data.num_rows, true)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, data.num_rows, true)); // 2. generate short key index (use cluster key) std::vector short_key_columns; for (const auto& cid : _tablet_schema->cluster_key_uids()) { @@ -1150,8 +1112,7 @@ Status VerticalSegmentWriter::_generate_key_index( } RETURN_IF_ERROR(_generate_short_key_index(short_key_columns, data.num_rows, short_key_pos)); } else if (_is_mow()) { - RETURN_IF_ERROR(_generate_primary_key_index(_key_coders, key_columns, seq_column, - data.num_rows, false)); + RETURN_IF_ERROR(_generate_primary_key_index(key_columns, seq_column, data.num_rows, false)); } else { // other tables RETURN_IF_ERROR(_generate_short_key_index(key_columns, data.num_rows, short_key_pos)); } @@ -1159,17 +1120,15 @@ Status VerticalSegmentWriter::_generate_key_index( } Status VerticalSegmentWriter::_generate_primary_key_index( - const std::vector& primary_key_coders, const std::vector& primary_key_columns, IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort) { if (!need_sort) { // mow table without cluster key std::string last_key; for (size_t pos = 0; pos < num_rows; pos++) { - // use _key_coders - std::string key = _full_encode_keys(primary_key_columns, pos); + std::string key = _key_encoder.full_encode(primary_key_columns, pos); _maybe_invalid_row_cache(key); if (_tablet_schema->has_sequence_col()) { - _encode_seq_column(seq_column, pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, pos); } DCHECK(key.compare(last_key) > 0) << "found duplicate key or key is not sorted! current key: " << key @@ -1181,12 +1140,12 @@ Status VerticalSegmentWriter::_generate_primary_key_index( // 1. generate primary keys in memory std::vector primary_keys; for (uint32_t pos = 0; pos < num_rows; pos++) { - std::string key = _full_encode_keys(primary_key_coders, primary_key_columns, pos); + std::string key = _key_encoder.full_encode_primary_keys(primary_key_columns, pos); _maybe_invalid_row_cache(key); if (_tablet_schema->has_sequence_col()) { - _encode_seq_column(seq_column, pos, &key); + _key_encoder.append_seq_suffix(&key, seq_column, pos); } - _encode_rowid(pos, &key); + _key_encoder.append_rowid_suffix(&key, pos); primary_keys.emplace_back(std::move(key)); } // 2. sort primary keys @@ -1207,9 +1166,8 @@ Status VerticalSegmentWriter::_generate_primary_key_index( Status VerticalSegmentWriter::_generate_short_key_index( std::vector& key_columns, size_t num_rows, const std::vector& short_key_pos) { - // use _key_coders - _set_min_key(_full_encode_keys(key_columns, 0)); - _set_max_key(_full_encode_keys(key_columns, num_rows - 1)); + _set_min_key(_key_encoder.full_encode(key_columns, 0)); + _set_max_key(_key_encoder.full_encode(key_columns, num_rows - 1)); DCHECK(Slice(_max_key.data(), _max_key.size()) .compare(Slice(_min_key.data(), _min_key.size())) >= 0) << "key is not sorted! min key: " << _min_key << ", max key: " << _max_key; @@ -1217,7 +1175,7 @@ Status VerticalSegmentWriter::_generate_short_key_index( key_columns.resize(_num_short_key_columns); std::string last_key; for (const auto pos : short_key_pos) { - std::string key = _encode_keys(key_columns, pos); + std::string key = _key_encoder.encode_short_keys(key_columns, pos); DCHECK(key.compare(last_key) >= 0) << "key is not sorted! current key: " << key << ", last key: " << last_key; RETURN_IF_ERROR(_short_key_index_builder->add_item(key)); @@ -1226,82 +1184,6 @@ Status VerticalSegmentWriter::_generate_short_key_index( return Status::OK(); } -void VerticalSegmentWriter::_encode_rowid(const uint32_t rowid, std::string* encoded_keys) { - encoded_keys->push_back(KEY_NORMAL_MARKER); - _rowid_coder->full_encode_ascending(&rowid, encoded_keys); -} - -std::string VerticalSegmentWriter::_full_encode_keys( - const std::vector& key_columns, size_t pos) { - assert(_key_index_size.size() == _num_sort_key_columns); - if (!(key_columns.size() == _num_sort_key_columns && - _key_coders.size() == _num_sort_key_columns)) { - LOG_INFO("key_columns.size()={}, _key_coders.size()={}, _num_sort_key_columns={}, ", - key_columns.size(), _key_coders.size(), _num_sort_key_columns); - } - assert(key_columns.size() == _num_sort_key_columns && - _key_coders.size() == _num_sort_key_columns); - return _full_encode_keys(_key_coders, key_columns, pos); -} - -std::string VerticalSegmentWriter::_full_encode_keys( - const std::vector& key_coders, - const std::vector& key_columns, size_t pos) { - assert(key_columns.size() == key_coders.size()); - - std::string encoded_keys; - size_t cid = 0; - for (const auto& column : key_columns) { - auto field = column->get_data_at(pos); - if (UNLIKELY(!field)) { - encoded_keys.push_back(KEY_NULL_FIRST_MARKER); - ++cid; - continue; - } - encoded_keys.push_back(KEY_NORMAL_MARKER); - DCHECK(key_coders[cid] != nullptr); - key_coders[cid]->full_encode_ascending(field, &encoded_keys); - ++cid; - } - return encoded_keys; -} - -void VerticalSegmentWriter::_encode_seq_column(const IOlapColumnDataAccessor* seq_column, - size_t pos, std::string* encoded_keys) { - const auto* field = seq_column->get_data_at(pos); - // To facilitate the use of the primary key index, encode the seq column - // to the minimum value of the corresponding length when the seq column - // is null - if (UNLIKELY(!field)) { - encoded_keys->push_back(KEY_NULL_FIRST_MARKER); - size_t seq_col_length = _tablet_schema->column(_tablet_schema->sequence_col_idx()).length(); - encoded_keys->append(seq_col_length, KEY_MINIMAL_MARKER); - return; - } - encoded_keys->push_back(KEY_NORMAL_MARKER); - _seq_coder->full_encode_ascending(field, encoded_keys); -} - -std::string VerticalSegmentWriter::_encode_keys( - const std::vector& key_columns, size_t pos) { - assert(key_columns.size() == _num_short_key_columns); - - std::string encoded_keys; - size_t cid = 0; - for (const auto& column : key_columns) { - auto field = column->get_data_at(pos); - if (UNLIKELY(!field)) { - encoded_keys.push_back(KEY_NULL_FIRST_MARKER); - ++cid; - continue; - } - encoded_keys.push_back(KEY_NORMAL_MARKER); - _key_coders[cid]->encode_ascending(field, _key_index_size[cid], &encoded_keys); - ++cid; - } - return encoded_keys; -} - // TODO(lingbin): Currently this function does not include the size of various indexes, // We should make this more precise. uint64_t VerticalSegmentWriter::_estimated_remaining_size() { diff --git a/be/src/storage/segment/vertical_segment_writer.h b/be/src/storage/segment/vertical_segment_writer.h index 1398cce94fdda7..ba381ea20c705e 100644 --- a/be/src/storage/segment/vertical_segment_writer.h +++ b/be/src/storage/segment/vertical_segment_writer.h @@ -31,6 +31,7 @@ #include "common/status.h" // Status #include "storage/index/index_file_writer.h" +#include "storage/key/row_key_encoder.h" #include "storage/olap_define.h" #include "storage/partial_update_info.h" #include "storage/segment/column_writer.h" @@ -147,18 +148,6 @@ class VerticalSegmentWriter { Status _write_footer(); Status _write_raw_data(const std::vector& slices); void _maybe_invalid_row_cache(const std::string& key) const; - std::string _encode_keys(const std::vector& key_columns, size_t pos); - // used for unique-key with merge on write and segment min_max key - std::string _full_encode_keys(const std::vector& key_columns, - size_t pos); - std::string _full_encode_keys(const std::vector& key_coders, - const std::vector& key_columns, - size_t pos); - // used for unique-key with merge on write - void _encode_seq_column(const IOlapColumnDataAccessor* seq_column, size_t pos, - std::string* encoded_keys); - // used for unique-key with merge on write tables with cluster keys - void _encode_rowid(const uint32_t rowid, std::string* encoded_keys); void _set_min_max_key(const Slice& key); void _set_min_key(const Slice& key); void _set_max_key(const Slice& key); @@ -194,7 +183,6 @@ class VerticalSegmentWriter { IOlapColumnDataAccessor* seq_column, std::map& cid_to_column); Status _generate_primary_key_index( - const std::vector& primary_key_coders, const std::vector& primary_key_columns, IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort); Status _generate_short_key_index(std::vector& key_columns, @@ -224,9 +212,6 @@ class VerticalSegmentWriter { SegmentFooterPB _footer; SegmentIndexFileCacheInfo _index_file_cache_info; - // for mow tables with cluster key, the sort key is the cluster keys not unique keys - // for other tables, the sort key is the keys - size_t _num_sort_key_columns; size_t _num_short_key_columns; std::unique_ptr _short_key_index_builder; @@ -236,12 +221,9 @@ class VerticalSegmentWriter { std::unique_ptr _olap_data_convertor; // used for building short key index or primary key index during vectorized write. - std::vector _key_coders; - // for mow table with cluster keys, this is primary keys - std::vector _primary_key_coders; - const KeyCoder* _seq_coder = nullptr; - const KeyCoder* _rowid_coder = nullptr; - std::vector _key_index_size; + // NOTE: must stay declared after _tablet_schema and _opts, the constructor + // init list reads both through _is_mow(). + RowKeyEncoder _key_encoder; size_t _short_key_row_pos = 0; // _num_rows_written means row count already written in this current column group diff --git a/be/test/storage/key/row_key_encoder_test.cpp b/be/test/storage/key/row_key_encoder_test.cpp new file mode 100644 index 00000000000000..eebc2e50b7c570 --- /dev/null +++ b/be/test/storage/key/row_key_encoder_test.cpp @@ -0,0 +1,637 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The test builds TabletSchema/TabletColumn objects field by field, the same +// way the existing storage unit tests do (tablet_schema_helper.cpp does the same). +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wkeyword-macro" +#define private public +#define protected public +#pragma clang diagnostic pop + +#include "storage/key/row_key_encoder.h" + +#include + +#include +#include +#include +#include + +#include "common/consts.h" +#include "core/block/block.h" +#include "storage/iterator/olap_data_convertor.h" +#include "storage/olap_common.h" +#include "storage/tablet/tablet_schema.h" +#include "storage/tablet/tablet_schema_helper.h" +#include "storage/utils.h" + +namespace doris { +namespace { + +constexpr uint8_t kNull = KeyConsts::KEY_NULL_FIRST_MARKER; // 0x01 +constexpr uint8_t kNormal = KeyConsts::KEY_NORMAL_MARKER; // 0x02 +constexpr uint8_t kMinimal = KeyConsts::KEY_MINIMAL_MARKER; // 0x00 + +// A hidden sequence column is identified by its reserved name. +TabletColumnPtr create_seq_col(int32_t uid) { + auto c = std::make_shared(); + c->_unique_id = uid; + c->_col_name = SEQUENCE_COL; + c->_type = FieldType::OLAP_FIELD_TYPE_INT; + c->_is_key = false; + c->_is_nullable = true; + c->_length = 4; + c->_index_length = 4; + return c; +} + +void fill_int(MutableColumns& cols, uint32_t cid, const std::vector>& vals) { + for (const auto& v : vals) { + if (v.has_value()) { + int32_t x = *v; + cols[cid]->insert_data(reinterpret_cast(&x), sizeof(x)); + } else { + cols[cid]->insert_default(); // NULL for a nullable column + } + } +} + +void fill_char(MutableColumns& cols, uint32_t cid, const std::vector& vals) { + for (const auto& s : vals) { + cols[cid]->insert_data(s.data(), s.size()); + } +} + +uint8_t byte_at(const std::string& s, size_t i) { + return static_cast(s[i]); +} + +// key(0), key(1), value(2): two int key columns. +TabletSchemaSPtr two_int_key_schema() { + auto s = std::make_shared(); + s->append_column(*create_int_key(0)); + s->append_column(*create_int_key(1)); + s->append_column(*create_int_value(2)); + s->_keys_type = DUP_KEYS; + s->_num_short_key_columns = 2; + return s; +} + +// char(8) key (index_length 1) + value: tests short-key truncation. +TabletSchemaSPtr char_key_schema() { + auto s = std::make_shared(); + s->append_column(*create_char_key(0, /*is_nullable=*/true, /*length=*/8)); + s->append_column(*create_int_value(1)); + s->_keys_type = DUP_KEYS; + s->_num_short_key_columns = 1; + return s; +} + +// key(0), value(1), seq(2): unique-key table with a sequence column. +TabletSchemaSPtr seq_schema() { + auto s = std::make_shared(); + s->append_column(*create_int_key(0)); + s->append_column(*create_int_value(1)); + s->append_column(*create_seq_col(2)); + s->_keys_type = UNIQUE_KEYS; + s->_num_short_key_columns = 1; + return s; +} + +// key(uid 0), value(uid 1) with the value column declared as the cluster key. +TabletSchemaSPtr cluster_key_schema() { + auto s = std::make_shared(); + s->append_column(*create_int_key(0)); + s->append_column(*create_int_value(1)); + s->_keys_type = UNIQUE_KEYS; + s->_cluster_key_uids = {1}; + s->_num_short_key_columns = 1; + return s; +} + +std::string to_hex(const std::string& s) { + static constexpr char kDigits[] = "0123456789abcdef"; + std::string out; + out.reserve(s.size() * 2); + for (unsigned char c : s) { + out.push_back(kDigits[c >> 4]); + out.push_back(kDigits[c & 0xf]); + } + return out; +} + +template +void fill_raw(MutableColumns& cols, uint32_t cid, T v) { + cols[cid]->insert_data(reinterpret_cast(&v), sizeof(v)); +} + +TabletColumnPtr make_key_column(int32_t uid, FieldType type, int32_t length, int32_t index_length, + int32_t precision = 0, int32_t frac = 0) { + auto c = std::make_shared(); + c->_unique_id = uid; + c->_col_name = "k" + std::to_string(uid); + c->_type = type; + c->_is_key = true; + c->_is_nullable = false; + c->_length = length; + c->_index_length = index_length; + c->_precision = precision; + c->_frac = frac; + return c; +} + +// One key column of the given type + one int value column, unique keys so the +// primary-key view exists. +TabletSchemaSPtr single_key_schema(const TabletColumnPtr& key_col) { + auto s = std::make_shared(); + s->append_column(*key_col); + s->append_column(*create_int_value(100)); + s->_keys_type = UNIQUE_KEYS; + s->_num_short_key_columns = 1; + return s; +} + +} // namespace + +// A small holder so the convertor (which the accessors point into) and the +// source block stay alive until after the encode calls. +class RowKeyEncoderTest : public testing::Test { +protected: + void build(const TabletSchemaSPtr& schema, size_t num_rows, + const std::function& fill) { + _schema = schema; + _num_rows = num_rows; + _block = schema->create_block(); + { + auto guard = _block.mutate_columns_scoped(); + fill(guard.mutable_columns()); + } + _convertor = std::make_unique(schema.get()); + _convertor->set_source_content(&_block, 0, num_rows); + } + + IOlapColumnDataAccessor* acc(uint32_t cid) { + auto [st, accessor] = _convertor->convert_column_data(cid); + EXPECT_TRUE(st.ok()) << st; + return accessor; + } + + // Encode one row through all three views and compare against literal hex + // goldens. The goldens pin the on-disk key byte layout: any coder change + // or CPU-architecture (endianness) drift must fail these expectations. + void check_golden(const TabletSchemaSPtr& schema, + const std::function& fill, const std::string& full_hex, + const std::string& primary_hex, const std::string& short_hex) { + build(schema, 1, fill); + RowKeyEncoder enc(*_schema, /*mow=*/true); + std::vector keys {acc(0)}; + EXPECT_EQ(to_hex(enc.full_encode(keys, 0)), full_hex); + EXPECT_EQ(to_hex(enc.full_encode_primary_keys(keys, 0)), primary_hex); + EXPECT_EQ(to_hex(enc.encode_short_keys(keys, 0)), short_hex); + } + + TabletSchemaSPtr _schema; + Block _block; + std::unique_ptr _convertor; + size_t _num_rows = 0; +}; + +// Each non-null key column is one marker byte + the encoded value. +TEST_F(RowKeyEncoderTest, FullEncodeIntKeyLayout) { + build(two_int_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {1}); + fill_int(c, 1, {2}); + fill_int(c, 2, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0), acc(1)}; + std::string k = enc.full_encode(keys, 0); + + ASSERT_EQ(k.size(), 1u + 4u + 1u + 4u); + EXPECT_EQ(byte_at(k, 0), kNormal); + EXPECT_EQ(byte_at(k, 5), kNormal); +} + +// A null key value is a single null marker with no value bytes. +TEST_F(RowKeyEncoderTest, NullKeyUsesNullMarker) { + build(two_int_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {std::nullopt}); + fill_int(c, 1, {7}); + fill_int(c, 2, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0), acc(1)}; + std::string k = enc.full_encode(keys, 0); + + ASSERT_EQ(k.size(), 1u + (1u + 4u)); + EXPECT_EQ(byte_at(k, 0), kNull); + EXPECT_EQ(byte_at(k, 1), kNormal); +} + +// The encoded key is a sortable byte string: byte-by-byte order over the +// encodings must match the multi-column key order, and nulls sort first. +TEST_F(RowKeyEncoderTest, FullEncodePreservesAscendingOrder) { + build(two_int_key_schema(), 5, [](MutableColumns& c) { + fill_int(c, 0, {std::nullopt, -5, -5, 2, 2}); + fill_int(c, 1, {0, 3, 9, -100, 7}); + fill_int(c, 2, {0, 0, 0, 0, 0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0), acc(1)}; + + std::vector encoded; + for (size_t pos = 0; pos < _num_rows; ++pos) { + encoded.push_back(enc.full_encode(keys, pos)); + } + for (size_t i = 0; i + 1 < encoded.size(); ++i) { + EXPECT_LT(encoded[i], encoded[i + 1]) << "rows " << i << " and " << i + 1; + } +} + +// Without cluster keys the sort key and the schema (primary) key are identical. +TEST_F(RowKeyEncoderTest, PrimaryKeysEqualFullEncodeWithoutClusterKey) { + build(two_int_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {3}); + fill_int(c, 1, {4}); + fill_int(c, 2, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/true); + std::vector keys {acc(0), acc(1)}; + EXPECT_EQ(enc.full_encode(keys, 0), enc.full_encode_primary_keys(keys, 0)); +} + +// With cluster keys the segment sorts by the cluster key columns, while the +// primary key index is still built over the schema key columns. +TEST_F(RowKeyEncoderTest, ClusterKeySortDiffersFromPrimaryKeys) { + build(cluster_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {5}); // schema key + fill_int(c, 1, {99}); // cluster key (and value) + }); + RowKeyEncoder enc(*_schema, /*mow=*/true); + ASSERT_EQ(enc.num_sort_key_columns(), 1u); + + std::vector sort_keys {acc(1)}; + std::vector primary_keys {acc(0)}; + EXPECT_NE(enc.full_encode(sort_keys, 0), enc.full_encode_primary_keys(primary_keys, 0)); +} + +// Short keys truncate each column to its index length; the full key keeps the +// whole value, so it is strictly longer for a char column with index_length 1. +TEST_F(RowKeyEncoderTest, ShortKeyTruncatesCharToIndexLength) { + build(char_key_schema(), 1, [](MutableColumns& c) { + fill_char(c, 0, {"abcdefgh"}); + fill_int(c, 1, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0)}; + + std::string full = enc.full_encode(keys, 0); + std::string short_key = enc.encode_short_keys(keys, 0); + + EXPECT_EQ(byte_at(short_key, 0), kNormal); + EXPECT_EQ(short_key.size(), 1u + 1u); // marker + index_length(1) + EXPECT_GT(full.size(), short_key.size()); +} + +// A null sequence value is encoded as the minimal value of the column length so +// it sorts before any real sequence value sharing the same key. +TEST_F(RowKeyEncoderTest, AppendSeqSuffixNullAndNormal) { + build(seq_schema(), 2, [](MutableColumns& c) { + fill_int(c, 0, {1, 1}); + fill_int(c, 1, {0, 0}); + fill_int(c, 2, {std::nullopt, 7}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/true); + std::vector keys {acc(0)}; + IOlapColumnDataAccessor* seq = acc(2); + + std::string null_seq = enc.full_encode(keys, 0); + size_t base = null_seq.size(); + enc.append_seq_suffix(&null_seq, seq, 0); + // null marker + 4 minimal bytes (seq column length) + ASSERT_EQ(null_seq.size(), base + 1u + 4u); + EXPECT_EQ(byte_at(null_seq, base), kNull); + EXPECT_EQ(byte_at(null_seq, base + 1), kMinimal); + + std::string normal_seq = enc.full_encode(keys, 1); + size_t base1 = normal_seq.size(); + enc.append_seq_suffix(&normal_seq, seq, 1); + EXPECT_EQ(byte_at(normal_seq, base1), kNormal); + + // same key, null sequence sorts before a real one + EXPECT_LT(null_seq, normal_seq); +} + +// The row id suffix keeps the encoded keys ordered by row id. +TEST_F(RowKeyEncoderTest, AppendRowidSuffixOrdersByRowid) { + build(cluster_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {5}); + fill_int(c, 1, {99}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/true); + std::vector sort_keys {acc(1)}; + + std::string base = enc.full_encode(sort_keys, 0); + std::string r1 = base; + enc.append_rowid_suffix(&r1, 1); + std::string r2 = base; + enc.append_rowid_suffix(&r2, 2); + + EXPECT_EQ(byte_at(r1, base.size()), kNormal); + EXPECT_GT(r1.size(), base.size()); + EXPECT_LT(r1, r2); +} + +// Short keys over fixed-length int columns are not truncated (index length == +// column length), so the short key over both key columns equals the full key. +TEST_F(RowKeyEncoderTest, ShortKeyIntColumnsEqualFullEncode) { + build(two_int_key_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {5}); + fill_int(c, 1, {6}); + fill_int(c, 2, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0), acc(1)}; + + std::string short_key = enc.encode_short_keys(keys, 0); + EXPECT_EQ(short_key.size(), (1u + 4u) + (1u + 4u)); // two non-truncated int columns + EXPECT_EQ(short_key, enc.full_encode(keys, 0)); +} + +// A null short-key column is a single null marker with no value bytes, exactly +// as in the full key. +TEST_F(RowKeyEncoderTest, ShortKeyNullColumnUsesNullMarker) { + build(char_key_schema(), 1, [](MutableColumns& c) { + c[0]->insert_default(); // NULL char key (the column is nullable) + fill_int(c, 1, {0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0)}; + + std::string short_key = enc.encode_short_keys(keys, 0); + ASSERT_EQ(short_key.size(), 1u); // marker only, value truncated away + EXPECT_EQ(byte_at(short_key, 0), kNull); +} + +// Short keys keep only a prefix and drop the rest: two char values sharing the +// first index_length byte collide in the short key, while their full keys still +// differ. +TEST_F(RowKeyEncoderTest, ShortKeyTruncationCollides) { + build(char_key_schema(), 2, [](MutableColumns& c) { + fill_char(c, 0, {"axxxxxxx", "ayyyyyyy"}); // share the first byte 'a' + fill_int(c, 1, {0, 0}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/false); + std::vector keys {acc(0)}; + + EXPECT_EQ(enc.encode_short_keys(keys, 0), enc.encode_short_keys(keys, 1)); // collide + EXPECT_NE(enc.full_encode(keys, 0), enc.full_encode(keys, 1)); // full differs +} + +// ---- Golden byte matrix over every key-column type coder ------------------- +// +// Every hex golden below is a literal snapshot of the encoded bytes. Format of +// a non-null key column: one KEY_NORMAL_MARKER byte (0x02) followed by the +// KeyCoder output — big-endian with the sign bit flipped for signed integrals +// and decimals, plain big-endian for unsigned types, raw bytes for strings +// (truncated to index_length in the short key). +// +// Not covered here: legacy v1 DATE/DATETIME/DECIMAL (only constructible +// through v1 value APIs; new tables use the v2 types pinned below), +// FLOAT/DOUBLE (not legal key columns), and UNSIGNED_INT/UNSIGNED_BIGINT +// (reachable only as the rowid suffix, pinned by GoldenSeqAndRowidSuffix). + +TEST_F(RowKeyEncoderTest, GoldenTinyInt) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_TINYINT, 1, 1)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "0285", "0285", "0285"); +} + +TEST_F(RowKeyEncoderTest, GoldenSmallInt) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_SMALLINT, 2, 2)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "028005", "028005", "028005"); +} + +TEST_F(RowKeyEncoderTest, GoldenInt) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_INT, 4, 4)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "0280000005", "0280000005", "0280000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenIntNegative) { + // -5: sign-flip + big-endian => 0x7ffffffb, sorts below the +5 golden above + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_INT, 4, 4)), + [](MutableColumns& c) { + fill_raw(c, 0, -5); + fill_int(c, 1, {0}); + }, + "027ffffffb", "027ffffffb", "027ffffffb"); +} + +TEST_F(RowKeyEncoderTest, GoldenBigInt) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_BIGINT, 8, 8)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "028000000000000005", "028000000000000005", "028000000000000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenLargeInt) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_LARGEINT, 16, 16)), + [](MutableColumns& c) { + fill_raw<__int128>(c, 0, 5); + fill_int(c, 1, {0}); + }, + "0280000000000000000000000000000005", "0280000000000000000000000000000005", + "0280000000000000000000000000000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenBool) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_BOOL, 1, 1)), + [](MutableColumns& c) { + fill_raw(c, 0, 1); + fill_int(c, 1, {0}); + }, + "0201", "0201", "0201"); +} + +TEST_F(RowKeyEncoderTest, GoldenDateV2) { + // DATEV2 stores a raw uint32; the coder is plain big-endian (no sign flip). + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_DATEV2, 4, 4)), + [](MutableColumns& c) { + fill_raw(c, 0, 0x01020304); + fill_int(c, 1, {0}); + }, + "0201020304", "0201020304", "0201020304"); +} + +TEST_F(RowKeyEncoderTest, GoldenDateTimeV2) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_DATETIMEV2, 8, 8)), + [](MutableColumns& c) { + fill_raw(c, 0, 0x0102030405060708ULL); + fill_int(c, 1, {0}); + }, + "020102030405060708", "020102030405060708", "020102030405060708"); +} + +TEST_F(RowKeyEncoderTest, GoldenDecimal32) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_DECIMAL32, 4, 4, 9, 2)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "0280000005", "0280000005", "0280000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenDecimal64) { + check_golden( + single_key_schema( + make_key_column(0, FieldType::OLAP_FIELD_TYPE_DECIMAL64, 8, 8, 18, 4)), + [](MutableColumns& c) { + fill_raw(c, 0, 5); + fill_int(c, 1, {0}); + }, + "028000000000000005", "028000000000000005", "028000000000000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenDecimal128I) { + check_golden( + single_key_schema( + make_key_column(0, FieldType::OLAP_FIELD_TYPE_DECIMAL128I, 16, 16, 38, 6)), + [](MutableColumns& c) { + fill_raw<__int128>(c, 0, 5); + fill_int(c, 1, {0}); + }, + "0280000000000000000000000000000005", "0280000000000000000000000000000005", + "0280000000000000000000000000000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenDecimal256) { + // Little-endian limb layout of wide::Int256 value 5 (x86-64); the coder + // sign-flips and emits the 32 bytes big-endian. + std::array raw {}; + raw[0] = 5; + check_golden( + single_key_schema( + make_key_column(0, FieldType::OLAP_FIELD_TYPE_DECIMAL256, 32, 32, 76, 8)), + [&raw](MutableColumns& c) { + c[0]->insert_data(reinterpret_cast(raw.data()), raw.size()); + fill_int(c, 1, {0}); + }, + "028000000000000000000000000000000000000000000000000000000000000005", + "028000000000000000000000000000000000000000000000000000000000000005", + "028000000000000000000000000000000000000000000000000000000000000005"); +} + +TEST_F(RowKeyEncoderTest, GoldenIpv4) { + // IPV4 stores a raw uint32 (127.0.0.1), plain big-endian, no sign flip. + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_IPV4, 4, 4)), + [](MutableColumns& c) { + fill_raw(c, 0, 0x7f000001); + fill_int(c, 1, {0}); + }, + "027f000001", "027f000001", "027f000001"); +} + +TEST_F(RowKeyEncoderTest, GoldenIpv6) { + // ::1 == uint128 value 1 + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_IPV6, 16, 16)), + [](MutableColumns& c) { + fill_raw(c, 0, 1); + fill_int(c, 1, {0}); + }, + "0200000000000000000000000000000001", "0200000000000000000000000000000001", + "0200000000000000000000000000000001"); +} + +TEST_F(RowKeyEncoderTest, GoldenChar) { + // CHAR(8) "abcdefgh": full/primary keep all bytes, the short key truncates + // to index_length(2). Covers the string coder in all three views. + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_CHAR, 8, 2)), + [](MutableColumns& c) { + fill_char(c, 0, {"abcdefgh"}); + fill_int(c, 1, {0}); + }, + "026162636465666768", "026162636465666768", "026162"); +} + +TEST_F(RowKeyEncoderTest, GoldenVarchar) { + // VARCHAR value "ab" shorter than index_length(4): the short key keeps all + // available bytes instead of failing. + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_VARCHAR, 16, 4)), + [](MutableColumns& c) { + fill_char(c, 0, {"ab"}); + fill_int(c, 1, {0}); + }, + "026162", "026162", "026162"); +} + +TEST_F(RowKeyEncoderTest, GoldenString) { + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_STRING, 1024, 4)), + [](MutableColumns& c) { + fill_char(c, 0, {"abcdef"}); + fill_int(c, 1, {0}); + }, + "02616263646566", "02616263646566", "0261626364"); +} + +// The two key suffixes: an int sequence value and the UNSIGNED_INT rowid. +TEST_F(RowKeyEncoderTest, GoldenSeqAndRowidSuffix) { + build(seq_schema(), 1, [](MutableColumns& c) { + fill_int(c, 0, {1}); + fill_int(c, 1, {0}); + fill_int(c, 2, {7}); + }); + RowKeyEncoder enc(*_schema, /*mow=*/true); + std::string seq_suffix; + enc.append_seq_suffix(&seq_suffix, acc(2), 0); + EXPECT_EQ(to_hex(seq_suffix), "0280000007"); + + RowKeyEncoder cluster_enc(*cluster_key_schema(), /*mow=*/true); + std::string rowid_suffix; + cluster_enc.append_rowid_suffix(&rowid_suffix, 3); + EXPECT_EQ(to_hex(rowid_suffix), "0200000003"); +} + +} // namespace doris diff --git a/be/test/storage/segment/test_segment_writer.h b/be/test/storage/segment/test_segment_writer.h index fcb385e255456c..9820c56e2f910a 100644 --- a/be/test/storage/segment/test_segment_writer.h +++ b/be/test/storage/segment/test_segment_writer.h @@ -107,7 +107,7 @@ class TestSegmentWriter : public SegmentWriter { RETURN_IF_ERROR(_column_writers[cid]->append(false, const_cast(ptr))); } std::string full_encoded_key; - row.encode_key(&full_encoded_key, _num_sort_key_columns); + row.encode_key(&full_encoded_key, _tablet_schema->num_key_columns()); if (_tablet_schema->has_sequence_col()) { full_encoded_key.push_back(KeyConsts::KEY_NORMAL_MARKER); auto cid = _tablet_schema->sequence_col_idx(); From 8802f5ea042363d0a60ff6ae4dda678f57a25fe4 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Sun, 12 Jul 2026 13:42:32 +0800 Subject: [PATCH 2/3] [test](storage) scope the private/protected macros to tablet_schema.h only Address review: the white-box macros were defined before every include of the test TU, so gtest, standard and other Doris headers were all parsed with rewritten access specifiers. Wrap only storage/tablet/tablet_schema.h (the one header the test needs white-box access to) and #undef right after, following the convention of segment_iterator_limit_opt_test.cpp. Co-Authored-By: Claude Fable 5 --- be/test/storage/key/row_key_encoder_test.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/be/test/storage/key/row_key_encoder_test.cpp b/be/test/storage/key/row_key_encoder_test.cpp index eebc2e50b7c570..0107e2af35f512 100644 --- a/be/test/storage/key/row_key_encoder_test.cpp +++ b/be/test/storage/key/row_key_encoder_test.cpp @@ -15,15 +15,23 @@ // specific language governing permissions and limitations // under the License. -// The test builds TabletSchema/TabletColumn objects field by field, the same -// way the existing storage unit tests do (tablet_schema_helper.cpp does the same). +// White-box access to TabletColumn/TabletSchema private fields: the test +// builds schema objects field by field, the same way the existing storage +// unit tests do (tablet_schema_helper.cpp does the same). The macros wrap +// only this include and are #undef-ed right away, so gtest, standard and +// other Doris headers below are parsed with their access specifiers intact. +#if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wkeyword-macro" +#endif #define private public #define protected public +#include "storage/tablet/tablet_schema.h" // IWYU pragma: keep +#undef private +#undef protected +#if defined(__clang__) #pragma clang diagnostic pop - -#include "storage/key/row_key_encoder.h" +#endif #include @@ -35,8 +43,8 @@ #include "common/consts.h" #include "core/block/block.h" #include "storage/iterator/olap_data_convertor.h" +#include "storage/key/row_key_encoder.h" #include "storage/olap_common.h" -#include "storage/tablet/tablet_schema.h" #include "storage/tablet/tablet_schema_helper.h" #include "storage/utils.h" From 98b579df6ede3036faf37e802c7e5a723853b1cd Mon Sep 17 00:00:00 2001 From: csun5285 Date: Mon, 13 Jul 2026 16:38:28 +0800 Subject: [PATCH 3/3] [test](storage) pin TIMESTAMPTZ in the RowKeyEncoder golden matrix TIMESTAMPTZ is a legal key column type (the FE key-type blacklist does not exclude it) with its own registered key coder (raw big-endian uint64), so the golden byte matrix must pin it like the other key types. Co-Authored-By: Claude Fable 5 --- be/test/storage/key/row_key_encoder_test.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/be/test/storage/key/row_key_encoder_test.cpp b/be/test/storage/key/row_key_encoder_test.cpp index 0107e2af35f512..628ec3d9dba8e6 100644 --- a/be/test/storage/key/row_key_encoder_test.cpp +++ b/be/test/storage/key/row_key_encoder_test.cpp @@ -517,6 +517,17 @@ TEST_F(RowKeyEncoderTest, GoldenDateTimeV2) { "020102030405060708", "020102030405060708", "020102030405060708"); } +TEST_F(RowKeyEncoderTest, GoldenTimestampTz) { + // TIMESTAMPTZ stores a raw uint64 (UTC-normalized), plain big-endian. + check_golden( + single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, 8, 8)), + [](MutableColumns& c) { + fill_raw(c, 0, 0x0102030405060708ULL); + fill_int(c, 1, {0}); + }, + "020102030405060708", "020102030405060708", "020102030405060708"); +} + TEST_F(RowKeyEncoderTest, GoldenDecimal32) { check_golden( single_key_schema(make_key_column(0, FieldType::OLAP_FIELD_TYPE_DECIMAL32, 4, 4, 9, 2)),