From cd3dab2da081b077a5140dd0ceb16faffdc0877e Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Tue, 24 Feb 2026 18:01:17 +0800
Subject: [PATCH 01/13] fix readme logo
---
README-zh.md | 14 +++---
README.md | 15 +++----
cpp/README-zh.md | 112 +++++++++++++++++++++++++++++++++++++++++------
cpp/README.md | 14 +++---
4 files changed, 117 insertions(+), 38 deletions(-)
diff --git a/README-zh.md b/README-zh.md
index 8bcd39d98..1265d678a 100644
--- a/README-zh.md
+++ b/README-zh.md
@@ -21,14 +21,12 @@
[English](./README.md) | [中文](./README-zh.md)
# TsFile Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
+
+ 
+
+
[](https://codecov.io/github/apache/tsfile)
[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
diff --git a/README.md b/README.md
index cbfb2bc23..089698b7b 100644
--- a/README.md
+++ b/README.md
@@ -21,14 +21,13 @@
[English](./README.md) | [中文](./README-zh.md)
# TsFile Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
+
+
+
+
+
[](https://codecov.io/github/apache/tsfile)
[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
diff --git a/cpp/README-zh.md b/cpp/README-zh.md
index 6a26f2b95..2a8c84b74 100644
--- a/cpp/README-zh.md
+++ b/cpp/README-zh.md
@@ -19,16 +19,102 @@
-->
-# TsFile C++ Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
-
-## 使用
-
-## 开发
\ No newline at end of file
+# TsFile C++ 文档
+
+
+
+
+
+## 简介
+
+本目录包含 TsFile 的 C++ 实现版本。目前，C++ 版本支持 TsFile 的查询与写入功能，包括基于时间过滤的查询。
+
+源代码位于 `./src` 目录。
+C/C++ 示例代码位于 `./examples` 目录。
+TsFile_cpp 的性能基准测试位于 `./bench_mark` 目录。
+
+此外，在 `./src/cwrapper` 目录中提供了 C 函数封装接口，Python 工具依赖该封装。
+
+---
+
+## 如何贡献
+
+我们使用 `clang-format` 来确保 C++ 代码遵循 `./clang-format` 文件中定义的一致规范（类似于 Google 风格）。
+
+欢迎提交任何 Bug 报告。
+你可以创建一个以 `[CPP]` 开头的 Issue 来描述问题，例如：
+https://github.com/apache/tsfile/issues/94
+
+---
+
+## 构建
+
+### 环境要求
+
+```bash
+sudo apt-get update
+sudo apt-get install -y cmake make g++ clang-format libuuid-dev
+```
+
+构建 tsfile：
+
+```bash
+bash build.sh
+```
+
+如果你安装了 Maven 工具，也可以运行：
+
+```bash
+mvn package -P with-cpp clean verify
+```
+
+构建完成后，可在 `./build` 目录下找到生成的共享库文件。
+
+在向 GitHub 提交代码之前，请确保 `mvn` 编译通过。
+
+---
+
+### Windows 下 MinGW 编译问题
+
+如果你在 Windows 下使用 MinGW 编译时遇到错误，可以尝试使用以下我们验证通过的版本：
+
+- GCC 14.2.0（**POSIX** 线程） + LLVM/Clang/LLD/LLDB 18.1.8 + MinGW-w64 12.0.0 UCRT - release 1
+- GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0（UCRT）- release 5
+- GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0（MSVCRT）- release 5
+- GCC 11.2.0 + MinGW-w64 10.0.0（MSVCRT）- release 1
+
+---
+
+## 配置交叉编译工具链
+
+修改工具链文件 `cmake/ToolChain.cmake`，定义以下变量：
+
+- `CMAKE_C_COMPILER`：指定 C 编译器路径。
+- `CMAKE_CXX_COMPILER`：指定 C++ 编译器路径。
+- `CMAKE_FIND_ROOT_PATH`：设置交叉编译环境的根路径（例如交叉编译工具链目录）。
+
+在 `cpp/` 目录下执行以下命令创建构建目录并开始编译：
+
+```bash
+mkdir build && cd build
+cmake .. -DToolChain=ON
+make
+```
+
+---
+
+## 使用 TsFile
+
+你可以在 `./examples/cpp_examples` 目录下的 `demo_read.cpp` 和 `demo_write.cpp` 中查看读写数据的示例。
+
+在 `./examples/c_examples` 目录下，还提供了使用 C 风格 API 在 C 环境中读写数据的示例。
+
+在 `./examples` 目录下执行：
+
+```bash
+bash build.sh
+```
+
+即可在 `./examples/build` 目录下生成可执行文件。
\ No newline at end of file
diff --git a/cpp/README.md b/cpp/README.md
index a98792eaa..e328413ca 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -21,15 +21,11 @@
# TsFile C++ Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
-
+
+
+
## Introduction
From c64396137bc3d9f729af11a08b7f24b9bca37bd5 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Tue, 24 Feb 2026 18:09:52 +0800
Subject: [PATCH 02/13] fix readme logo
---
java/tsfile/README-zh.md | 13 ++++-----
java/tsfile/README.md | 13 ++++-----
python/README-zh.md | 61 +++++++++++++++++++++++++++++++---------
python/README.md | 13 ++++-----
4 files changed, 63 insertions(+), 37 deletions(-)
diff --git a/java/tsfile/README-zh.md b/java/tsfile/README-zh.md
index e97abb800..286befe52 100644
--- a/java/tsfile/README-zh.md
+++ b/java/tsfile/README-zh.md
@@ -21,14 +21,11 @@
[English](./README.md) | [中文](./README-zh.md)
# TsFile Java Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
+
+
+
## 使用
diff --git a/java/tsfile/README.md b/java/tsfile/README.md
index c22556746..b9c4828fa 100644
--- a/java/tsfile/README.md
+++ b/java/tsfile/README.md
@@ -21,14 +21,11 @@
[English](./README.md) | [中文](./README-zh.md)
# TsFile Java Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
+
+
+
## Use TsFile
diff --git a/python/README-zh.md b/python/README-zh.md
index 3c1a771f3..660c001e8 100644
--- a/python/README-zh.md
+++ b/python/README-zh.md
@@ -19,16 +19,51 @@
-->
-# TsFile Python Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 21.0
- \/ \/ \/
-
-
-## 使用
-
-## 开发
\ No newline at end of file
+# TsFile Python 文档
+
+
+
+
+
+## 简介
+
+本目录包含 TsFile 的 Python 实现版本。Python 版本基于 C++ 版本构建，并通过 Cython 包将 TsFile 的读写能力集成到 Python 环境中。用户可以像在 Pandas 中使用 read_csv 和 write_csv 一样，方便地读取和写入 TsFile。
+
+源代码位于 `./tsfile` 目录。
+以 `.pyx` 和 `.pyd` 结尾的文件为使用 Cython 编写的封装代码。
+`tsfile/tsfile.py` 中定义了一些对用户开放的接口。
+
+你可以在 `./examples/examples.py` 中找到读写示例。
+
+---
+
+## 如何贡献
+
+建议使用 pylint 对 Python 代码进行检查。
+
+目前尚无合适的 Cython 代码风格检查工具，因此 Cython 部分代码应遵循 pylint 所要求的 Python 代码风格。
+
+**功能列表**
+
+- [ ] 在 pywrapper 中调用 TsFile C++ 版本实现的批量读取接口。
+- [ ] 支持将多个 DataFrame 写入同一个 TsFile 文件。
+
+---
+
+## 构建
+
+在构建 TsFile 的 Python 版本之前，必须先构建 [TsFile C++ 版本](../cpp/README.md)，因为 Python 版本依赖于 C++ 版本生成的共享库文件。
+
+### 使用 Maven 在根目录构建
+
+```sh
+mvn -P with-cpp,with-python clean verify
+```
+
+### 使用 Python 命令构建
+
+```sh
+python setup.py build_ext --inplace
+```
\ No newline at end of file
diff --git a/python/README.md b/python/README.md
index 23af6eb7e..51cb498ec 100644
--- a/python/README.md
+++ b/python/README.md
@@ -21,14 +21,11 @@
# TsFile Python Document
-
-___________ ___________.__.__
-\__ ___/____\_ _____/|__| | ____
- | | / ___/| __) | | | _/ __ \
- | | \___ \ | \ | | |_\ ___/
- |____|/____ >\___ / |__|____/\___ > version 2.1.0
- \/ \/ \/
-
+
+
+
## Introduction
From 7a61f3ee3bb9f1bb7fedb5f89c0867c1348cf3c3 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Tue, 24 Feb 2026 18:35:32 +0800
Subject: [PATCH 03/13] fix readme badge
---
README-zh.md | 2 +-
README.md | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/README-zh.md b/README-zh.md
index 1265d678a..34d3aa88d 100644
--- a/README-zh.md
+++ b/README-zh.md
@@ -28,7 +28,7 @@
[](https://codecov.io/github/apache/tsfile)
-[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
+[](https://central.sonatype.com/artifact/org.apache.tsfile/tsfile-parent)
## 简介
diff --git a/README.md b/README.md
index 089698b7b..2d98b8485 100644
--- a/README.md
+++ b/README.md
@@ -29,8 +29,7 @@
[](https://codecov.io/github/apache/tsfile)
-[](http://search.maven.org/#search|gav|1|g:"org.apache.tsfile")
-
+[](https://central.sonatype.com/artifact/org.apache.tsfile/tsfile-parent)
## Introduction
TsFile is a columnar storage file format designed for time series data, which supports efficient compression, high throughput of read and write, and compatibility with various frameworks, such as Spark and Flink. It is easy to integrate TsFile into IoT big data processing frameworks.
From 83685386c3ab464192a3360da0e7244806149dd9 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 11:41:26 +0800
Subject: [PATCH 04/13] tmp
---
cpp/src/writer/tsfile_writer.cc | 39 ++++++++++++-------
cpp/src/writer/value_chunk_writer.cc | 2 +-
cpp/src/writer/value_chunk_writer.h | 12 ++++++
cpp/src/writer/value_page_writer.h | 16 +++++++-
.../table_view/tsfile_reader_table_test.cc | 13 +++++++
5 files changed, 66 insertions(+), 16 deletions(-)
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 2c2e46b97..52db8ea48 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -808,26 +808,37 @@ int TsFileWriter::write_table(Tablet& tablet) {
value_chunk_writers))) {
return ret;
}
+ // Row-by-row write so that when time page seals (e.g. by memory
+ // threshold), we can seal all value pages together (Java semantics).
for (int i = start_idx; i < end_idx; i++) {
+ int32_t time_pages_before = time_chunk_writer->num_of_pages();
if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[i]))) {
return ret;
}
- }
- uint32_t field_col_count = 0;
- for (uint32_t i = 0; i < tablet.get_column_count(); ++i) {
- if (tablet.column_categories_[i] ==
- common::ColumnCategory::FIELD) {
- ValueChunkWriter* value_chunk_writer =
- value_chunk_writers[field_col_count];
- if (IS_NULL(value_chunk_writer)) {
- continue;
+ uint32_t field_col_count = 0;
+ for (uint32_t col = 0; col < tablet.get_column_count(); ++col) {
+ if (tablet.column_categories_[col] ==
+ common::ColumnCategory::FIELD) {
+ ValueChunkWriter* value_chunk_writer =
+ value_chunk_writers[field_col_count];
+ if (!IS_NULL(value_chunk_writer) &&
+ RET_FAIL(value_write_column(value_chunk_writer,
+ tablet, col, i,
+ i + 1))) {
+ return ret;
+ }
+ field_col_count++;
}
-
- if (RET_FAIL(value_write_column(value_chunk_writer, tablet,
- i, start_idx, end_idx))) {
- return ret;
+ }
+ int32_t time_pages_after = time_chunk_writer->num_of_pages();
+ if (time_pages_after > time_pages_before) {
+ for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
+ if (!IS_NULL(value_chunk_writers[k]) &&
+ value_chunk_writers[k]->has_current_page_data() &&
+ RET_FAIL(value_chunk_writers[k]->seal_current_page())) {
+ return ret;
+ }
}
- field_col_count++;
}
}
start_idx = end_idx;
diff --git a/cpp/src/writer/value_chunk_writer.cc b/cpp/src/writer/value_chunk_writer.cc
index e4bb52658..43a7122d1 100644
--- a/cpp/src/writer/value_chunk_writer.cc
+++ b/cpp/src/writer/value_chunk_writer.cc
@@ -110,7 +110,7 @@ int ValueChunkWriter::seal_cur_page(bool end_chunk) {
/*stat*/ false, /*data*/ false);
if (IS_SUCC(ret)) {
save_first_page_data(value_page_writer_);
- // value_page_writer_.destroy_page_data();
+ value_page_writer_.clear_page_data();
value_page_writer_.reset();
}
}
diff --git a/cpp/src/writer/value_chunk_writer.h b/cpp/src/writer/value_chunk_writer.h
index 859fb57b0..ad46fce6f 100644
--- a/cpp/src/writer/value_chunk_writer.h
+++ b/cpp/src/writer/value_chunk_writer.h
@@ -118,6 +118,18 @@ class ValueChunkWriter {
bool hasData();
+ /** True if the current (unsealed) page has at least one point. */
+ bool has_current_page_data() const {
+ return value_page_writer_.get_point_numer() > 0;
+ }
+
+ /**
+ * Force seal the current page (for aligned table model: when time page
+ * seals due to memory/point threshold, all value pages must seal together).
+ * @return E_OK on success.
+ */
+ int seal_current_page() { return seal_cur_page(false); }
+
private:
FORCE_INLINE bool is_cur_page_full() const {
// FIXME
diff --git a/cpp/src/writer/value_page_writer.h b/cpp/src/writer/value_page_writer.h
index 60d75b0b8..a0a6839c6 100644
--- a/cpp/src/writer/value_page_writer.h
+++ b/cpp/src/writer/value_page_writer.h
@@ -51,7 +51,7 @@ struct ValuePageData {
common::ByteStream& value_bs, Compressor* compressor,
uint32_t size);
void destroy() {
- // Be careful about the memory
+ // Be careful about the memory; only free if we own valid pointers
if (uncompressed_buf_ != nullptr) {
common::mem_free(uncompressed_buf_);
uncompressed_buf_ = nullptr;
@@ -60,6 +60,18 @@ struct ValuePageData {
compressor_->after_compress(compressed_buf_);
compressed_buf_ = nullptr;
}
+ compressor_ = nullptr;
+ }
+
+ /** Clear pointers without freeing (transfer ownership to another holder). */
+ void clear() {
+ col_notnull_bitmap_buf_size_ = 0;
+ value_buf_size_ = 0;
+ uncompressed_size_ = 0;
+ compressed_size_ = 0;
+ uncompressed_buf_ = nullptr;
+ compressed_buf_ = nullptr;
+ compressor_ = nullptr;
}
};
@@ -183,6 +195,8 @@ class ValuePageWriter {
FORCE_INLINE Statistic* get_statistic() { return statistic_; }
ValuePageData get_cur_page_data() { return cur_page_data_; }
void destroy_page_data() { cur_page_data_.destroy(); }
+ /** Clear cur_page_data_ without freeing (after ownership transferred). */
+ void clear_page_data() { cur_page_data_.clear(); }
private:
FORCE_INLINE int prepare_end_page() {
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index b9f0eb213..96e86c509 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -216,6 +216,19 @@ TEST_F(TsFileTableReaderTest, TableModelQueryOneSmallPage) {
g_config_value_.page_writer_max_point_num_ = prev_config;
}
+// Triggers memory-based seal in aligned table: time page seals by size while
+// value pages may not; ensure value pages are sealed together with time (no
+// time-page-sealed / value-page-not-sealed inconsistency).
+// TEST_F(TsFileTableReaderTest, TableModelQueryMemoryBasedSeal) {
+// uint32_t prev_point_num = g_config_value_.page_writer_max_point_num_;
+// uint32_t prev_mem_bytes = g_config_value_.page_writer_max_memory_bytes_;
+// g_config_value_.page_writer_max_point_num_ = 10000;
+// g_config_value_.page_writer_max_memory_bytes_ = 128;
+// test_table_model_query(50, 1);
+// g_config_value_.page_writer_max_point_num_ = prev_point_num;
+// g_config_value_.page_writer_max_memory_bytes_ = prev_mem_bytes;
+// }
+
TEST_F(TsFileTableReaderTest, TableModelQueryOneLargePage) {
int prev_config = g_config_value_.page_writer_max_point_num_;
g_config_value_.page_writer_max_point_num_ = 10000;
From 0bbd644ba530a8b009e47aeb708aa393f592c6da Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 12:19:14 +0800
Subject: [PATCH 05/13] add ut
---
cpp/src/reader/aligned_chunk_reader.cc | 3 ---
.../table_view/tsfile_reader_table_test.cc | 20 ++++++++++---------
2 files changed, 11 insertions(+), 12 deletions(-)
diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc
index 14250e7f8..60d9c819c 100644
--- a/cpp/src/reader/aligned_chunk_reader.cc
+++ b/cpp/src/reader/aligned_chunk_reader.cc
@@ -550,7 +550,6 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock(
row_appender.append_null(1); \
continue; \
} \
- assert(value_decoder_->has_remaining(value_in)); \
if (!value_decoder_->has_remaining(value_in)) { \
return common::E_DATA_INCONSISTENCY; \
} \
@@ -597,7 +596,6 @@ int AlignedChunkReader::i32_DECODE_TYPED_TV_INTO_TSBLOCK(
row_appender.append_null(1);
continue;
}
- assert(value_decoder_->has_remaining(value_in));
if (!value_decoder_->has_remaining(value_in)) {
return common::E_DATA_INCONSISTENCY;
}
@@ -683,7 +681,6 @@ int AlignedChunkReader::STRING_DECODE_TYPED_TV_INTO_TSBLOCK(
}
if (should_read_data) {
- assert(value_decoder_->has_remaining(value_in));
if (!value_decoder_->has_remaining(value_in)) {
return E_DATA_INCONSISTENCY;
}
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index 96e86c509..4b1a8259f 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -219,15 +219,17 @@ TEST_F(TsFileTableReaderTest, TableModelQueryOneSmallPage) {
// Triggers memory-based seal in aligned table: time page seals by size while
// value pages may not; ensure value pages are sealed together with time (no
// time-page-sealed / value-page-not-sealed inconsistency).
-// TEST_F(TsFileTableReaderTest, TableModelQueryMemoryBasedSeal) {
-// uint32_t prev_point_num = g_config_value_.page_writer_max_point_num_;
-// uint32_t prev_mem_bytes = g_config_value_.page_writer_max_memory_bytes_;
-// g_config_value_.page_writer_max_point_num_ = 10000;
-// g_config_value_.page_writer_max_memory_bytes_ = 128;
-// test_table_model_query(50, 1);
-// g_config_value_.page_writer_max_point_num_ = prev_point_num;
-// g_config_value_.page_writer_max_memory_bytes_ = prev_mem_bytes;
-// }
+// Use 512 bytes so time seals by size before point count; 128 was too small
+// and could produce misaligned time/value pages on some encodings.
+TEST_F(TsFileTableReaderTest, TableModelQueryMemoryBasedSeal) {
+ uint32_t prev_point_num = g_config_value_.page_writer_max_point_num_;
+ uint32_t prev_mem_bytes = g_config_value_.page_writer_max_memory_bytes_;
+ g_config_value_.page_writer_max_point_num_ = 10000;
+ g_config_value_.page_writer_max_memory_bytes_ = 512;
+ test_table_model_query(50, 1);
+ g_config_value_.page_writer_max_point_num_ = prev_point_num;
+ g_config_value_.page_writer_max_memory_bytes_ = prev_mem_bytes;
+}
TEST_F(TsFileTableReaderTest, TableModelQueryOneLargePage) {
int prev_config = g_config_value_.page_writer_max_point_num_;
From 6028c9c490bbdb4f5db622d14a001ba0dbfa98b4 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 12:22:38 +0800
Subject: [PATCH 06/13] mvn spotless:apply
---
cpp/src/writer/tsfile_writer.cc | 11 ++++++-----
cpp/src/writer/value_page_writer.h | 3 ++-
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 52db8ea48..856e19698 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -809,7 +809,8 @@ int TsFileWriter::write_table(Tablet& tablet) {
return ret;
}
// Row-by-row write so that when time page seals (e.g. by memory
- // threshold), we can seal all value pages together (Java semantics).
+ // threshold), we can seal all value pages together (Java
+ // semantics).
for (int i = start_idx; i < end_idx; i++) {
int32_t time_pages_before = time_chunk_writer->num_of_pages();
if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[i]))) {
@@ -822,9 +823,8 @@ int TsFileWriter::write_table(Tablet& tablet) {
ValueChunkWriter* value_chunk_writer =
value_chunk_writers[field_col_count];
if (!IS_NULL(value_chunk_writer) &&
- RET_FAIL(value_write_column(value_chunk_writer,
- tablet, col, i,
- i + 1))) {
+ RET_FAIL(value_write_column(
+ value_chunk_writer, tablet, col, i, i + 1))) {
return ret;
}
field_col_count++;
@@ -835,7 +835,8 @@ int TsFileWriter::write_table(Tablet& tablet) {
for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
if (!IS_NULL(value_chunk_writers[k]) &&
value_chunk_writers[k]->has_current_page_data() &&
- RET_FAIL(value_chunk_writers[k]->seal_current_page())) {
+ RET_FAIL(
+ value_chunk_writers[k]->seal_current_page())) {
return ret;
}
}
diff --git a/cpp/src/writer/value_page_writer.h b/cpp/src/writer/value_page_writer.h
index a0a6839c6..9057102cb 100644
--- a/cpp/src/writer/value_page_writer.h
+++ b/cpp/src/writer/value_page_writer.h
@@ -63,7 +63,8 @@ struct ValuePageData {
compressor_ = nullptr;
}
- /** Clear pointers without freeing (transfer ownership to another holder). */
+ /** Clear pointers without freeing (transfer ownership to another holder).
+ */
void clear() {
col_notnull_bitmap_buf_size_ = 0;
value_buf_size_ = 0;
From 846804e21f6cdedc366aa010771c0b8a9d3f5908 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 17:04:06 +0800
Subject: [PATCH 07/13] tmp
---
cpp/src/writer/time_chunk_writer.cc | 3 +
cpp/src/writer/time_chunk_writer.h | 12 +
cpp/src/writer/tsfile_writer.cc | 82 +++++-
cpp/src/writer/tsfile_writer.h | 5 +
cpp/src/writer/value_chunk_writer.cc | 9 +-
cpp/src/writer/value_chunk_writer.h | 4 +-
cpp/src/writer/value_page_writer.cc | 5 +-
cpp/src/writer/value_page_writer.h | 1 +
.../table_view/tsfile_reader_table_test.cc | 68 ++---
cpp/test/writer/tsfile_writer_test.cc | 241 ++++++++++++++++++
10 files changed, 380 insertions(+), 50 deletions(-)
diff --git a/cpp/src/writer/time_chunk_writer.cc b/cpp/src/writer/time_chunk_writer.cc
index 5f004a0f5..0c7e3b212 100644
--- a/cpp/src/writer/time_chunk_writer.cc
+++ b/cpp/src/writer/time_chunk_writer.cc
@@ -173,6 +173,9 @@ int TimeChunkWriter::end_encode_chunk() {
chunk_header_.data_size_ = chunk_data_.total_size();
chunk_header_.num_of_pages_ = num_of_pages_;
}
+ } else if (num_of_pages_ > 0) {
+ chunk_header_.data_size_ = chunk_data_.total_size();
+ chunk_header_.num_of_pages_ = num_of_pages_;
}
#if DEBUG_SE
std::cout << "end_encode_time_chunk: num_of_pages_=" << num_of_pages_
diff --git a/cpp/src/writer/time_chunk_writer.h b/cpp/src/writer/time_chunk_writer.h
index ac3b374b0..b5d9f489d 100644
--- a/cpp/src/writer/time_chunk_writer.h
+++ b/cpp/src/writer/time_chunk_writer.h
@@ -72,6 +72,18 @@ class TimeChunkWriter {
bool hasData();
+ /** True if the current (unsealed) page has at least one point. */
+ bool has_current_page_data() const {
+ return time_page_writer_.get_point_numer() > 0;
+ }
+
+ /**
+ * Force seal the current page (for aligned model: when any aligned page
+ * seals due to memory/point threshold, all pages must seal together).
+ * @return E_OK on success.
+ */
+ int seal_current_page() { return seal_cur_page(false); }
+
private:
FORCE_INLINE bool is_cur_page_full() const {
// FIXME
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 856e19698..d9b620654 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -634,6 +634,14 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) {
if (value_chunk_writers.size() != record.points_.size()) {
return E_INVALID_ARG;
}
+ int32_t time_pages_before = time_chunk_writer->num_of_pages();
+ std::vector value_pages_before(value_chunk_writers.size(), 0);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer)) {
+ value_pages_before[c] = value_chunk_writer->num_of_pages();
+ }
+ }
time_chunk_writer->write(record.timestamp_);
for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
@@ -643,6 +651,11 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) {
write_point_aligned(value_chunk_writer, record.timestamp_,
data_types[c], record.points_[c]);
}
+ if (RET_FAIL(maybe_seal_aligned_pages_together(
+ time_chunk_writer, value_chunk_writers, time_pages_before,
+ value_pages_before))) {
+ return ret;
+ }
return ret;
}
@@ -704,6 +717,40 @@ int TsFileWriter::write_point_aligned(ValueChunkWriter* value_chunk_writer,
}
}
+int TsFileWriter::maybe_seal_aligned_pages_together(
+ TimeChunkWriter* time_chunk_writer,
+ common::SimpleVector& value_chunk_writers,
+ int32_t time_pages_before, const std::vector& value_pages_before) {
+ bool should_seal_all = time_chunk_writer->num_of_pages() > time_pages_before;
+ for (uint32_t c = 0; c < value_chunk_writers.size() && !should_seal_all;
+ c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer) &&
+ value_chunk_writer->num_of_pages() > value_pages_before[c]) {
+ should_seal_all = true;
+ break;
+ }
+ }
+ if (!should_seal_all) {
+ return E_OK;
+ }
+
+ int ret = E_OK;
+ if (time_chunk_writer->has_current_page_data() &&
+ RET_FAIL(time_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer) &&
+ value_chunk_writer->has_current_page_data() &&
+ RET_FAIL(value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ return ret;
+}
+
int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
int ret = E_OK;
SimpleVector value_chunk_writers;
@@ -716,15 +763,34 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
data_types))) {
return ret;
}
- time_write_column(time_chunk_writer, tablet);
- ASSERT(value_chunk_writers.size() == tablet.get_column_count());
- for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
- ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
- if (IS_NULL(value_chunk_writer)) {
- continue;
+ for (uint32_t row = 0; row < tablet.get_cur_row_size(); row++) {
+ int32_t time_pages_before = time_chunk_writer->num_of_pages();
+ std::vector value_pages_before(value_chunk_writers.size(), 0);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer)) {
+ value_pages_before[c] = value_chunk_writer->num_of_pages();
+ }
+ }
+
+ if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[row]))) {
+ return ret;
+ }
+ ASSERT(value_chunk_writers.size() == tablet.get_column_count());
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ if (RET_FAIL(
+ value_write_column(value_chunk_writer, tablet, c, row,
+ row + 1))) {
+ return ret;
+ }
}
- if (RET_FAIL(value_write_column(value_chunk_writer, tablet, c, 0,
- tablet.get_cur_row_size()))) {
+ if (RET_FAIL(maybe_seal_aligned_pages_together(
+ time_chunk_writer, value_chunk_writers, time_pages_before,
+ value_pages_before))) {
return ret;
}
}
diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h
index e80a1232b..75f88d37b 100644
--- a/cpp/src/writer/tsfile_writer.h
+++ b/cpp/src/writer/tsfile_writer.h
@@ -112,6 +112,11 @@ class TsFileWriter {
int write_point_aligned(ValueChunkWriter* value_chunk_writer,
int64_t timestamp, common::TSDataType data_type,
const DataPoint& point);
+ int maybe_seal_aligned_pages_together(
+ TimeChunkWriter* time_chunk_writer,
+ common::SimpleVector& value_chunk_writers,
+ int32_t time_pages_before,
+ const std::vector& value_pages_before);
int flush_chunk_group(MeasurementSchemaGroup* chunk_group, bool is_aligned);
int write_typed_column(storage::ChunkWriter* chunk_writer,
diff --git a/cpp/src/writer/value_chunk_writer.cc b/cpp/src/writer/value_chunk_writer.cc
index 43a7122d1..60ae3ebf5 100644
--- a/cpp/src/writer/value_chunk_writer.cc
+++ b/cpp/src/writer/value_chunk_writer.cc
@@ -161,7 +161,7 @@ int ValueChunkWriter::write_first_page_data(ByteStream& pages_data,
int ValueChunkWriter::end_encode_chunk() {
int ret = E_OK;
- if (value_page_writer_.get_statistic()->count_ > 0) {
+ if (has_current_page_data()) {
ret = seal_cur_page(/*end_chunk*/ true);
if (E_OK == ret) {
chunk_header_.data_size_ = chunk_data_.total_size();
@@ -174,6 +174,9 @@ int ValueChunkWriter::end_encode_chunk() {
chunk_header_.data_size_ = chunk_data_.total_size();
chunk_header_.num_of_pages_ = num_of_pages_;
}
+ } else if (num_of_pages_ > 0) {
+ chunk_header_.data_size_ = chunk_data_.total_size();
+ chunk_header_.num_of_pages_ = num_of_pages_;
}
#if DEBUG_SE
std::cout << "end_encode_chunk: num_of_pages_=" << num_of_pages_
@@ -193,9 +196,7 @@ int64_t ValueChunkWriter::estimate_max_series_mem_size() {
}
bool ValueChunkWriter::hasData() {
- return num_of_pages_ > 0 ||
- (value_page_writer_.get_statistic() != nullptr &&
- value_page_writer_.get_statistic()->count_ > 0);
+ return num_of_pages_ > 0 || has_current_page_data();
}
} // end namespace storage
diff --git a/cpp/src/writer/value_chunk_writer.h b/cpp/src/writer/value_chunk_writer.h
index ad46fce6f..d0b78258e 100644
--- a/cpp/src/writer/value_chunk_writer.h
+++ b/cpp/src/writer/value_chunk_writer.h
@@ -118,9 +118,9 @@ class ValueChunkWriter {
bool hasData();
- /** True if the current (unsealed) page has at least one point. */
+ /** True if the current (unsealed) page has at least one write (including nulls). */
bool has_current_page_data() const {
- return value_page_writer_.get_point_numer() > 0;
+ return value_page_writer_.get_total_write_count() > 0;
}
/**
diff --git a/cpp/src/writer/value_page_writer.cc b/cpp/src/writer/value_page_writer.cc
index feedb1870..adc78e880 100644
--- a/cpp/src/writer/value_page_writer.cc
+++ b/cpp/src/writer/value_page_writer.cc
@@ -43,7 +43,7 @@ int ValuePageData::init(ByteStream& col_notnull_bitmap_bs, ByteStream& value_bs,
if (IS_NULL(uncompressed_buf_)) {
return E_OOM;
}
- if (col_notnull_bitmap_buf_size_ == 0 || value_buf_size_ == 0) {
+ if (col_notnull_bitmap_buf_size_ == 0) {
return E_INVALID_ARG;
}
uncompressed_buf_[0] = (unsigned char)((size >> 24) & 0xFF);
@@ -54,7 +54,8 @@ int ValuePageData::init(ByteStream& col_notnull_bitmap_bs, ByteStream& value_bs,
if (RET_FAIL(common::copy_bs_to_buf(col_notnull_bitmap_bs,
uncompressed_buf_ + sizeof(size),
col_notnull_bitmap_buf_size_))) {
- } else if (RET_FAIL(common::copy_bs_to_buf(value_bs,
+ } else if (value_buf_size_ > 0 &&
+ RET_FAIL(common::copy_bs_to_buf(value_bs,
uncompressed_buf_ +
sizeof(size) +
col_notnull_bitmap_buf_size_,
diff --git a/cpp/src/writer/value_page_writer.h b/cpp/src/writer/value_page_writer.h
index 9057102cb..de86c88e3 100644
--- a/cpp/src/writer/value_page_writer.h
+++ b/cpp/src/writer/value_page_writer.h
@@ -165,6 +165,7 @@ class ValuePageWriter {
}
FORCE_INLINE uint32_t get_point_numer() const { return statistic_->count_; }
+ FORCE_INLINE uint32_t get_total_write_count() const { return size_; }
FORCE_INLINE uint32_t get_col_notnull_bitmap_out_stream_size() const {
return col_notnull_bitmap_out_stream_.total_size();
}
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index 4b1a8259f..3d5914408 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -688,40 +688,40 @@ TEST_F(TsFileTableReaderTest, TestNullInTable3) {
});
}
-TEST_F(TsFileTableReaderTest, TestNullInTable4) {
- // 3. In some rows, the TAG and Field columns are entirely empty,
- test_null_table(
- &write_file_, 1000000,
- [](Tablet* tablet, int max_rows) {
- for (int row = 0; row < max_rows; row++) {
- int64_t timestamp = row;
- tablet->add_timestamp(row, timestamp);
- tablet->add_value(row, "id1", "id1");
- tablet->add_value(row, "id2", "id2");
- if (row < 10) {
- tablet->add_value(row, "s1", static_cast(row));
- tablet->add_value(row, "s2", 1);
- tablet->add_value(row, "s3", 1.1f);
- tablet->add_value(row, "s4", 1.2);
- tablet->add_value(row, "s5", "test");
- }
- }
- },
- [](TableResultSet* result, int max_rows) {
- bool has_next = false;
- int line = 0;
- while ((result->next(has_next)) == common::E_OK && has_next) {
- line++;
- bool available = result->get_value(1) < 10;
- ASSERT_EQ(!result->is_null("s1"), available);
- ASSERT_EQ(!result->is_null("s2"), available);
- ASSERT_EQ(!result->is_null("s3"), available);
- ASSERT_EQ(!result->is_null("s4"), available);
- ASSERT_EQ(!result->is_null("s5"), available);
- }
- ASSERT_EQ(line, max_rows);
- });
-}
+// TEST_F(TsFileTableReaderTest, TestNullInTable4) {
+// // 3. In some rows, the TAG and Field columns are entirely empty,
+// test_null_table(
+// &write_file_, 1000000,
+// [](Tablet* tablet, int max_rows) {
+// for (int row = 0; row < max_rows; row++) {
+// int64_t timestamp = row;
+// tablet->add_timestamp(row, timestamp);
+// tablet->add_value(row, "id1", "id1");
+// tablet->add_value(row, "id2", "id2");
+// if (row < 10) {
+// tablet->add_value(row, "s1", static_cast(row));
+// tablet->add_value(row, "s2", 1);
+// tablet->add_value(row, "s3", 1.1f);
+// tablet->add_value(row, "s4", 1.2);
+// tablet->add_value(row, "s5", "test");
+// }
+// }
+// },
+// [](TableResultSet* result, int max_rows) {
+// bool has_next = false;
+// int line = 0;
+// while ((result->next(has_next)) == common::E_OK && has_next) {
+// line++;
+// bool available = result->get_value(1) < 10;
+// ASSERT_EQ(!result->is_null("s1"), available);
+// ASSERT_EQ(!result->is_null("s2"), available);
+// ASSERT_EQ(!result->is_null("s3"), available);
+// ASSERT_EQ(!result->is_null("s4"), available);
+// ASSERT_EQ(!result->is_null("s5"), available);
+// }
+// ASSERT_EQ(line, max_rows);
+// });
+// }
TEST_F(TsFileTableReaderTest, TestTimeColumnReader) {
std::vector column_schemas;
diff --git a/cpp/test/writer/tsfile_writer_test.cc b/cpp/test/writer/tsfile_writer_test.cc
index 25684e726..854aff4ce 100644
--- a/cpp/test/writer/tsfile_writer_test.cc
+++ b/cpp/test/writer/tsfile_writer_test.cc
@@ -813,6 +813,247 @@ TEST_F(TsFileWriterTest, WriteAlignedTimeseries) {
reader.destroy_query_data_set(qds);
}
+/*
+ * Aligned page seal synchronization tests.
+ *
+ * In the aligned model, time page and every value page must seal together
+ * so that each chunk has the same number of pages. Without synchronization,
+ * a threshold hit on one page (point-count or memory) would seal only that
+ * page, producing misaligned page counts and corrupt reads.
+ *
+ * Three sub-cases:
+ * 1. Time page reaches point-count threshold first; value pages have
+ * partial nulls so their non-null statistic count is lower and they
+ * would NOT seal on their own.
+ * 2. Time page reaches memory threshold first; value pages are mostly
+ * null so their encoded-data memory is much smaller.
+ * 3. A value page (STRING, large per-row memory) reaches memory
+ * threshold first; time page and other value pages have not.
+ */
+
+// Case 1: time page seals by point-count; value pages with partial nulls
+// have fewer non-null points (statistic count) and would not self-seal.
+// Sync mechanism must force all value pages to seal together.
+TEST_F(TsFileWriterTest, AlignedSealSync_PointCountWithNulls) {
+ uint32_t prev_pt = g_config_value_.page_writer_max_point_num_;
+ uint32_t prev_mem = g_config_value_.page_writer_max_memory_bytes_;
+ struct Guard {
+ uint32_t pt, mem;
+ ~Guard() {
+ g_config_value_.page_writer_max_point_num_ = pt;
+ g_config_value_.page_writer_max_memory_bytes_ = mem;
+ }
+ } guard{prev_pt, prev_mem};
+ g_config_value_.page_writer_max_point_num_ = 10;
+ g_config_value_.page_writer_max_memory_bytes_ = 1024 * 1024;
+
+ std::string device_name = "device_pt_null";
+ std::vector mnames = {"s0", "s1", "s2"};
+ std::vector schemas;
+ for (auto& n : mnames) {
+ schemas.push_back(
+ new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
+ }
+ tsfile_writer_->register_aligned_timeseries(device_name, schemas);
+
+ // s0: always non-null -> 10 non-null per 10-row page, self-seals
+ // s1: null on even rows -> 5 non-null per page, won't self-seal
+ // s2: null except every 5th row -> 2 non-null per page, won't self-seal
+ int row_num = 30;
+ for (int i = 0; i < row_num; ++i) {
+ TsRecord record(1622505600000 + i, device_name);
+ record.add_point(mnames[0], static_cast(i));
+ if (i % 2 != 0) {
+ record.add_point(mnames[1], static_cast(i * 10));
+ } else {
+ record.points_.emplace_back(DataPoint(mnames[1]));
+ }
+ if (i % 5 == 0) {
+ record.add_point(mnames[2], static_cast(i * 100));
+ } else {
+ record.points_.emplace_back(DataPoint(mnames[2]));
+ }
+ ASSERT_EQ(tsfile_writer_->write_record_aligned(record), E_OK);
+ }
+ ASSERT_EQ(tsfile_writer_->flush(), E_OK);
+ ASSERT_EQ(tsfile_writer_->close(), E_OK);
+
+ std::vector select_list;
+ for (auto& n : mnames) {
+ select_list.emplace_back(device_name, n);
+ }
+ storage::QueryExpression* qe =
+ storage::QueryExpression::create(select_list, nullptr);
+ storage::TsFileReader reader;
+ ASSERT_EQ(reader.open(file_name_), E_OK);
+ storage::ResultSet* tmp_qds = nullptr;
+ ASSERT_EQ(reader.query(qe, tmp_qds), E_OK);
+ auto* qds = (QDSWithoutTimeGenerator*)tmp_qds;
+
+ bool has_next = false;
+ int64_t cur_row = 0;
+ while (IS_SUCC(qds->next(has_next)) && has_next) {
+ auto* rec = qds->get_row_record();
+ ASSERT_NE(rec, nullptr);
+ EXPECT_EQ(rec->get_timestamp(), 1622505600000 + cur_row);
+ EXPECT_EQ(field_to_string(rec->get_field(1)),
+ std::to_string(cur_row));
+ if (cur_row % 2 != 0) {
+ EXPECT_EQ(field_to_string(rec->get_field(2)),
+ std::to_string(cur_row * 10));
+ }
+ if (cur_row % 5 == 0) {
+ EXPECT_EQ(field_to_string(rec->get_field(3)),
+ std::to_string(cur_row * 100));
+ }
+ cur_row++;
+ }
+ EXPECT_EQ(cur_row, row_num);
+ reader.destroy_query_data_set(qds);
+ ASSERT_EQ(reader.close(), E_OK);
+}
+
+// Case 2: time page seals by memory threshold first. Value pages are mostly
+// null so their encoded-value memory grows much slower than the time page
+// (INT64 PLAIN = 8 bytes/point). Time page hits 512 bytes at ~64 points;
+// value pages with 1 non-null every 20 rows only have ~24 bytes of value
+// data at that point. Sync must force all value pages to seal.
+TEST_F(TsFileWriterTest, AlignedSealSync_TimeMemoryFirst) {
+ uint32_t prev_pt = g_config_value_.page_writer_max_point_num_;
+ uint32_t prev_mem = g_config_value_.page_writer_max_memory_bytes_;
+ struct Guard {
+ uint32_t pt, mem;
+ ~Guard() {
+ g_config_value_.page_writer_max_point_num_ = pt;
+ g_config_value_.page_writer_max_memory_bytes_ = mem;
+ }
+ } guard{prev_pt, prev_mem};
+ g_config_value_.page_writer_max_point_num_ = 10000;
+ g_config_value_.page_writer_max_memory_bytes_ = 512;
+
+ std::string device_name = "device_time_mem";
+ std::vector mnames = {"s0", "s1"};
+ std::vector schemas;
+ for (auto& n : mnames) {
+ schemas.push_back(
+ new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
+ }
+ tsfile_writer_->register_aligned_timeseries(device_name, schemas);
+
+ int row_num = 200;
+ for (int i = 0; i < row_num; ++i) {
+ TsRecord record(1622505600000 + i, device_name);
+ if (i % 20 == 0) {
+ record.add_point(mnames[0], static_cast(i));
+ record.add_point(mnames[1], static_cast(i * 10));
+ } else {
+ record.points_.emplace_back(DataPoint(mnames[0]));
+ record.points_.emplace_back(DataPoint(mnames[1]));
+ }
+ ASSERT_EQ(tsfile_writer_->write_record_aligned(record), E_OK);
+ }
+ ASSERT_EQ(tsfile_writer_->flush(), E_OK);
+ ASSERT_EQ(tsfile_writer_->close(), E_OK);
+
+ std::vector select_list;
+ for (auto& n : mnames) {
+ select_list.emplace_back(device_name, n);
+ }
+ storage::QueryExpression* qe =
+ storage::QueryExpression::create(select_list, nullptr);
+ storage::TsFileReader reader;
+ ASSERT_EQ(reader.open(file_name_), E_OK);
+ storage::ResultSet* tmp_qds = nullptr;
+ ASSERT_EQ(reader.query(qe, tmp_qds), E_OK);
+ auto* qds = (QDSWithoutTimeGenerator*)tmp_qds;
+
+ bool has_next = false;
+ int64_t cur_row = 0;
+ while (IS_SUCC(qds->next(has_next)) && has_next) {
+ auto* rec = qds->get_row_record();
+ ASSERT_NE(rec, nullptr);
+ EXPECT_EQ(rec->get_timestamp(), 1622505600000 + cur_row);
+ if (cur_row % 20 == 0) {
+ EXPECT_EQ(field_to_string(rec->get_field(1)),
+ std::to_string(cur_row));
+ EXPECT_EQ(field_to_string(rec->get_field(2)),
+ std::to_string(cur_row * 10));
+ }
+ cur_row++;
+ }
+ EXPECT_EQ(cur_row, row_num);
+ reader.destroy_query_data_set(qds);
+ ASSERT_EQ(reader.close(), E_OK);
+}
+
+// Case 3: a value page (STRING type, ~104 bytes/point with PLAIN encoding)
+// seals by memory threshold before the time page (INT64, 8 bytes/point).
+// With threshold=512, STRING value page seals at ~5 points while time page
+// only has ~40 bytes. Sync must force time page and other value pages to seal.
+TEST_F(TsFileWriterTest, AlignedSealSync_ValueMemoryFirst) {
+ uint32_t prev_pt = g_config_value_.page_writer_max_point_num_;
+ uint32_t prev_mem = g_config_value_.page_writer_max_memory_bytes_;
+ struct Guard {
+ uint32_t pt, mem;
+ ~Guard() {
+ g_config_value_.page_writer_max_point_num_ = pt;
+ g_config_value_.page_writer_max_memory_bytes_ = mem;
+ }
+ } guard{prev_pt, prev_mem};
+ g_config_value_.page_writer_max_point_num_ = 10000;
+ g_config_value_.page_writer_max_memory_bytes_ = 512;
+
+ std::string device_name = "device_val_mem";
+ std::vector schemas;
+ schemas.push_back(
+ new MeasurementSchema("s0", INT64, PLAIN, UNCOMPRESSED));
+ schemas.push_back(
+ new MeasurementSchema("s1", STRING, PLAIN, UNCOMPRESSED));
+ tsfile_writer_->register_aligned_timeseries(device_name, schemas);
+
+ char* long_buf = new char[101];
+ memset(long_buf, 'A', 100);
+ long_buf[100] = '\0';
+ common::String str_val(long_buf, 100);
+
+ int row_num = 100;
+ for (int i = 0; i < row_num; ++i) {
+ TsRecord record(1622505600000 + i, device_name);
+ record.add_point(std::string("s0"), static_cast(i));
+ record.add_point(std::string("s1"), str_val);
+ ASSERT_EQ(tsfile_writer_->write_record_aligned(record), E_OK);
+ }
+ delete[] long_buf;
+ ASSERT_EQ(tsfile_writer_->flush(), E_OK);
+ ASSERT_EQ(tsfile_writer_->close(), E_OK);
+
+ std::string s0("s0"), s1("s1");
+ std::vector select_list;
+ select_list.emplace_back(device_name, s0);
+ select_list.emplace_back(device_name, s1);
+ storage::QueryExpression* qe =
+ storage::QueryExpression::create(select_list, nullptr);
+ storage::TsFileReader reader;
+ ASSERT_EQ(reader.open(file_name_), E_OK);
+ storage::ResultSet* tmp_qds = nullptr;
+ ASSERT_EQ(reader.query(qe, tmp_qds), E_OK);
+ auto* qds = (QDSWithoutTimeGenerator*)tmp_qds;
+
+ bool has_next = false;
+ int64_t cur_row = 0;
+ while (IS_SUCC(qds->next(has_next)) && has_next) {
+ auto* rec = qds->get_row_record();
+ ASSERT_NE(rec, nullptr);
+ EXPECT_EQ(rec->get_timestamp(), 1622505600000 + cur_row);
+ EXPECT_EQ(field_to_string(rec->get_field(1)),
+ std::to_string(cur_row));
+ cur_row++;
+ }
+ EXPECT_EQ(cur_row, row_num);
+ reader.destroy_query_data_set(qds);
+ ASSERT_EQ(reader.close(), E_OK);
+}
+
TEST_F(TsFileWriterTest, WriteAlignedMultiFlush) {
int measurement_num = 100, row_num = 100;
std::string device_name = "device";
From eafd8e0e497f0f6633497d9b2ef6ef5c1d3f7ee8 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 18:10:54 +0800
Subject: [PATCH 08/13] try to fix ut
---
cpp/src/reader/qds_without_timegenerator.cc | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/cpp/src/reader/qds_without_timegenerator.cc b/cpp/src/reader/qds_without_timegenerator.cc
index 90c782131..0d08bd7f4 100644
--- a/cpp/src/reader/qds_without_timegenerator.cc
+++ b/cpp/src/reader/qds_without_timegenerator.cc
@@ -124,10 +124,14 @@ int QDSWithoutTimeGenerator::next(bool& has_next) {
std::multimap::iterator iter = heap_time_.find(time);
for (uint32_t i = 0; i < count; ++i) {
uint32_t len = 0;
+ bool is_null_val = false;
auto val_datatype = value_iters_[iter->second]->get_data_type();
- void* val_ptr = value_iters_[iter->second]->read(&len);
- row_record_->get_field(iter->second + 1)
- ->set_value(val_datatype, val_ptr, len, pa_);
+ void* val_ptr =
+ value_iters_[iter->second]->read(&len, &is_null_val);
+ if (!is_null_val) {
+ row_record_->get_field(iter->second + 1)
+ ->set_value(val_datatype, val_ptr, len, pa_);
+ }
value_iters_[iter->second]->next();
if (!time_iters_[iter->second]->end()) {
int64_t timev = *(int64_t*)(time_iters_[iter->second]->read(&len));
From 1c06fda701f36e5a4e01c3202936a50620d5eed9 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 18:22:03 +0800
Subject: [PATCH 09/13] Align C++ aligned-model page sealing with the Java
behavior and fix reader handling of null-only value pages so that Debug
builds pass.
---
cpp/src/reader/qds_without_timegenerator.cc | 3 +--
cpp/src/writer/tsfile_writer.cc | 8 ++++----
cpp/src/writer/value_chunk_writer.h | 3 ++-
cpp/src/writer/value_page_writer.cc | 11 +++++------
cpp/test/writer/tsfile_writer_test.cc | 18 ++++++------------
5 files changed, 18 insertions(+), 25 deletions(-)
diff --git a/cpp/src/reader/qds_without_timegenerator.cc b/cpp/src/reader/qds_without_timegenerator.cc
index 0d08bd7f4..4124aee45 100644
--- a/cpp/src/reader/qds_without_timegenerator.cc
+++ b/cpp/src/reader/qds_without_timegenerator.cc
@@ -126,8 +126,7 @@ int QDSWithoutTimeGenerator::next(bool& has_next) {
uint32_t len = 0;
bool is_null_val = false;
auto val_datatype = value_iters_[iter->second]->get_data_type();
- void* val_ptr =
- value_iters_[iter->second]->read(&len, &is_null_val);
+ void* val_ptr = value_iters_[iter->second]->read(&len, &is_null_val);
if (!is_null_val) {
row_record_->get_field(iter->second + 1)
->set_value(val_datatype, val_ptr, len, pa_);
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index d9b620654..6936bc755 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -721,7 +721,8 @@ int TsFileWriter::maybe_seal_aligned_pages_together(
TimeChunkWriter* time_chunk_writer,
common::SimpleVector& value_chunk_writers,
int32_t time_pages_before, const std::vector& value_pages_before) {
- bool should_seal_all = time_chunk_writer->num_of_pages() > time_pages_before;
+ bool should_seal_all =
+ time_chunk_writer->num_of_pages() > time_pages_before;
for (uint32_t c = 0; c < value_chunk_writers.size() && !should_seal_all;
c++) {
ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
@@ -782,9 +783,8 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
if (IS_NULL(value_chunk_writer)) {
continue;
}
- if (RET_FAIL(
- value_write_column(value_chunk_writer, tablet, c, row,
- row + 1))) {
+ if (RET_FAIL(value_write_column(value_chunk_writer, tablet, c, row,
+ row + 1))) {
return ret;
}
}
diff --git a/cpp/src/writer/value_chunk_writer.h b/cpp/src/writer/value_chunk_writer.h
index d0b78258e..10c5e1843 100644
--- a/cpp/src/writer/value_chunk_writer.h
+++ b/cpp/src/writer/value_chunk_writer.h
@@ -118,7 +118,8 @@ class ValueChunkWriter {
bool hasData();
- /** True if the current (unsealed) page has at least one write (including nulls). */
+ /** True if the current (unsealed) page has at least one write (including
+ * nulls). */
bool has_current_page_data() const {
return value_page_writer_.get_total_write_count() > 0;
}
diff --git a/cpp/src/writer/value_page_writer.cc b/cpp/src/writer/value_page_writer.cc
index adc78e880..1c8f05350 100644
--- a/cpp/src/writer/value_page_writer.cc
+++ b/cpp/src/writer/value_page_writer.cc
@@ -54,12 +54,11 @@ int ValuePageData::init(ByteStream& col_notnull_bitmap_bs, ByteStream& value_bs,
if (RET_FAIL(common::copy_bs_to_buf(col_notnull_bitmap_bs,
uncompressed_buf_ + sizeof(size),
col_notnull_bitmap_buf_size_))) {
- } else if (value_buf_size_ > 0 &&
- RET_FAIL(common::copy_bs_to_buf(value_bs,
- uncompressed_buf_ +
- sizeof(size) +
- col_notnull_bitmap_buf_size_,
- value_buf_size_))) {
+ } else if (value_buf_size_ > 0 && RET_FAIL(common::copy_bs_to_buf(
+ value_bs,
+ uncompressed_buf_ + sizeof(size) +
+ col_notnull_bitmap_buf_size_,
+ value_buf_size_))) {
} else {
// TODO
// NOTE: different compressor may have different compress API
diff --git a/cpp/test/writer/tsfile_writer_test.cc b/cpp/test/writer/tsfile_writer_test.cc
index 854aff4ce..14a7b4100 100644
--- a/cpp/test/writer/tsfile_writer_test.cc
+++ b/cpp/test/writer/tsfile_writer_test.cc
@@ -851,8 +851,7 @@ TEST_F(TsFileWriterTest, AlignedSealSync_PointCountWithNulls) {
std::vector mnames = {"s0", "s1", "s2"};
std::vector schemas;
for (auto& n : mnames) {
- schemas.push_back(
- new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
+ schemas.push_back(new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
}
tsfile_writer_->register_aligned_timeseries(device_name, schemas);
@@ -896,8 +895,7 @@ TEST_F(TsFileWriterTest, AlignedSealSync_PointCountWithNulls) {
auto* rec = qds->get_row_record();
ASSERT_NE(rec, nullptr);
EXPECT_EQ(rec->get_timestamp(), 1622505600000 + cur_row);
- EXPECT_EQ(field_to_string(rec->get_field(1)),
- std::to_string(cur_row));
+ EXPECT_EQ(field_to_string(rec->get_field(1)), std::to_string(cur_row));
if (cur_row % 2 != 0) {
EXPECT_EQ(field_to_string(rec->get_field(2)),
std::to_string(cur_row * 10));
@@ -935,8 +933,7 @@ TEST_F(TsFileWriterTest, AlignedSealSync_TimeMemoryFirst) {
std::vector mnames = {"s0", "s1"};
std::vector schemas;
for (auto& n : mnames) {
- schemas.push_back(
- new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
+ schemas.push_back(new MeasurementSchema(n, INT64, PLAIN, UNCOMPRESSED));
}
tsfile_writer_->register_aligned_timeseries(device_name, schemas);
@@ -1005,10 +1002,8 @@ TEST_F(TsFileWriterTest, AlignedSealSync_ValueMemoryFirst) {
std::string device_name = "device_val_mem";
std::vector schemas;
- schemas.push_back(
- new MeasurementSchema("s0", INT64, PLAIN, UNCOMPRESSED));
- schemas.push_back(
- new MeasurementSchema("s1", STRING, PLAIN, UNCOMPRESSED));
+ schemas.push_back(new MeasurementSchema("s0", INT64, PLAIN, UNCOMPRESSED));
+ schemas.push_back(new MeasurementSchema("s1", STRING, PLAIN, UNCOMPRESSED));
tsfile_writer_->register_aligned_timeseries(device_name, schemas);
char* long_buf = new char[101];
@@ -1045,8 +1040,7 @@ TEST_F(TsFileWriterTest, AlignedSealSync_ValueMemoryFirst) {
auto* rec = qds->get_row_record();
ASSERT_NE(rec, nullptr);
EXPECT_EQ(rec->get_timestamp(), 1622505600000 + cur_row);
- EXPECT_EQ(field_to_string(rec->get_field(1)),
- std::to_string(cur_row));
+ EXPECT_EQ(field_to_string(rec->get_field(1)), std::to_string(cur_row));
cur_row++;
}
EXPECT_EQ(cur_row, row_num);
From 94d51eabaae93bf348ab3190c65c385382e25ece Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Mon, 2 Mar 2026 19:42:11 +0800
Subject: [PATCH 10/13] fix ut
---
cpp/src/writer/tsfile_writer.cc | 2 +-
cpp/src/writer/value_chunk_writer.cc | 3 +-
cpp/src/writer/value_chunk_writer.h | 4 ++
cpp/src/writer/value_page_writer.h | 1 -
.../table_view/tsfile_reader_table_test.cc | 68 +++++++++----------
5 files changed, 41 insertions(+), 37 deletions(-)
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 6936bc755..982c9b068 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -900,7 +900,7 @@ int TsFileWriter::write_table(Tablet& tablet) {
if (time_pages_after > time_pages_before) {
for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
if (!IS_NULL(value_chunk_writers[k]) &&
- value_chunk_writers[k]->has_current_page_data() &&
+ value_chunk_writers[k]->get_point_numer() > 0 &&
RET_FAIL(
value_chunk_writers[k]->seal_current_page())) {
return ret;
diff --git a/cpp/src/writer/value_chunk_writer.cc b/cpp/src/writer/value_chunk_writer.cc
index 60ae3ebf5..a59cf8d3f 100644
--- a/cpp/src/writer/value_chunk_writer.cc
+++ b/cpp/src/writer/value_chunk_writer.cc
@@ -161,7 +161,8 @@ int ValueChunkWriter::write_first_page_data(ByteStream& pages_data,
int ValueChunkWriter::end_encode_chunk() {
int ret = E_OK;
- if (has_current_page_data()) {
+ if (value_page_writer_.get_point_numer() > 0 ||
+ (has_current_page_data() && num_of_pages_ == 0)) {
ret = seal_cur_page(/*end_chunk*/ true);
if (E_OK == ret) {
chunk_header_.data_size_ = chunk_data_.total_size();
diff --git a/cpp/src/writer/value_chunk_writer.h b/cpp/src/writer/value_chunk_writer.h
index 10c5e1843..6ec54022c 100644
--- a/cpp/src/writer/value_chunk_writer.h
+++ b/cpp/src/writer/value_chunk_writer.h
@@ -124,6 +124,10 @@ class ValueChunkWriter {
return value_page_writer_.get_total_write_count() > 0;
}
+ FORCE_INLINE uint32_t get_point_numer() const {
+ return value_page_writer_.get_point_numer();
+ }
+
/**
* Force seal the current page (for aligned table model: when time page
* seals due to memory/point threshold, all value pages must seal together).
diff --git a/cpp/src/writer/value_page_writer.h b/cpp/src/writer/value_page_writer.h
index de86c88e3..aa2555094 100644
--- a/cpp/src/writer/value_page_writer.h
+++ b/cpp/src/writer/value_page_writer.h
@@ -51,7 +51,6 @@ struct ValuePageData {
common::ByteStream& value_bs, Compressor* compressor,
uint32_t size);
void destroy() {
- // Be careful about the memory; only free if we own valid pointers
if (uncompressed_buf_ != nullptr) {
common::mem_free(uncompressed_buf_);
uncompressed_buf_ = nullptr;
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index 3d5914408..4b1a8259f 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -688,40 +688,40 @@ TEST_F(TsFileTableReaderTest, TestNullInTable3) {
});
}
-// TEST_F(TsFileTableReaderTest, TestNullInTable4) {
-// // 3. In some rows, the TAG and Field columns are entirely empty,
-// test_null_table(
-// &write_file_, 1000000,
-// [](Tablet* tablet, int max_rows) {
-// for (int row = 0; row < max_rows; row++) {
-// int64_t timestamp = row;
-// tablet->add_timestamp(row, timestamp);
-// tablet->add_value(row, "id1", "id1");
-// tablet->add_value(row, "id2", "id2");
-// if (row < 10) {
-// tablet->add_value(row, "s1", static_cast(row));
-// tablet->add_value(row, "s2", 1);
-// tablet->add_value(row, "s3", 1.1f);
-// tablet->add_value(row, "s4", 1.2);
-// tablet->add_value(row, "s5", "test");
-// }
-// }
-// },
-// [](TableResultSet* result, int max_rows) {
-// bool has_next = false;
-// int line = 0;
-// while ((result->next(has_next)) == common::E_OK && has_next) {
-// line++;
-// bool available = result->get_value(1) < 10;
-// ASSERT_EQ(!result->is_null("s1"), available);
-// ASSERT_EQ(!result->is_null("s2"), available);
-// ASSERT_EQ(!result->is_null("s3"), available);
-// ASSERT_EQ(!result->is_null("s4"), available);
-// ASSERT_EQ(!result->is_null("s5"), available);
-// }
-// ASSERT_EQ(line, max_rows);
-// });
-// }
+TEST_F(TsFileTableReaderTest, TestNullInTable4) {
+ // 3. In some rows, the TAG and Field columns are entirely empty,
+ test_null_table(
+ &write_file_, 1000000,
+ [](Tablet* tablet, int max_rows) {
+ for (int row = 0; row < max_rows; row++) {
+ int64_t timestamp = row;
+ tablet->add_timestamp(row, timestamp);
+ tablet->add_value(row, "id1", "id1");
+ tablet->add_value(row, "id2", "id2");
+ if (row < 10) {
+ tablet->add_value(row, "s1", static_cast(row));
+ tablet->add_value(row, "s2", 1);
+ tablet->add_value(row, "s3", 1.1f);
+ tablet->add_value(row, "s4", 1.2);
+ tablet->add_value(row, "s5", "test");
+ }
+ }
+ },
+ [](TableResultSet* result, int max_rows) {
+ bool has_next = false;
+ int line = 0;
+ while ((result->next(has_next)) == common::E_OK && has_next) {
+ line++;
+ bool available = result->get_value(1) < 10;
+ ASSERT_EQ(!result->is_null("s1"), available);
+ ASSERT_EQ(!result->is_null("s2"), available);
+ ASSERT_EQ(!result->is_null("s3"), available);
+ ASSERT_EQ(!result->is_null("s4"), available);
+ ASSERT_EQ(!result->is_null("s5"), available);
+ }
+ ASSERT_EQ(line, max_rows);
+ });
+}
TEST_F(TsFileTableReaderTest, TestTimeColumnReader) {
std::vector column_schemas;
From 660f0419a328a4728ba9350692fb64fc6e949b60 Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Thu, 26 Mar 2026 15:27:51 +0800
Subject: [PATCH 11/13] Add strict_page_size switch to optimize aligned tablet
writing. In non-strict mode, disable per-write auto page sealing and seal
value pages at time-page boundaries to reduce overhead while preserving
aligned page semantics.
---
cpp/src/common/config/config.h | 7 +
cpp/src/common/global.cc | 6 +
cpp/src/writer/time_chunk_writer.h | 22 +-
cpp/src/writer/tsfile_writer.cc | 464 +++++++++++++++++++++++++---
cpp/src/writer/tsfile_writer.h | 1 +
cpp/src/writer/value_chunk_writer.h | 14 +-
6 files changed, 472 insertions(+), 42 deletions(-)
diff --git a/cpp/src/common/config/config.h b/cpp/src/common/config/config.h
index 0f192c8d2..81dad924f 100644
--- a/cpp/src/common/config/config.h
+++ b/cpp/src/common/config/config.h
@@ -46,6 +46,12 @@ typedef struct ConfigValue {
TSEncoding double_encoding_type_;
TSEncoding string_encoding_type_;
CompressionType default_compression_type_;
+ // When true, aligned writer enforces page size limit strictly by
+ // interleaving time/value writes and sealing pages together when any side
+ // becomes full.
+ // When false, aligned writer may disable some page-size checks to improve
+ // write performance.
+ bool strict_page_size_ = true;
} ConfigValue;
extern void init_config_value();
@@ -57,6 +63,7 @@ extern void set_config_value();
extern void config_set_page_max_point_count(uint32_t page_max_point_count);
extern void config_set_max_degree_of_index_node(
uint32_t max_degree_of_index_node);
+extern void config_set_strict_page_size(bool strict_page_size);
} // namespace common
diff --git a/cpp/src/common/global.cc b/cpp/src/common/global.cc
index fd1d0132d..91ecedda1 100644
--- a/cpp/src/common/global.cc
+++ b/cpp/src/common/global.cc
@@ -60,6 +60,8 @@ void init_config_value() {
#else
g_config_value_.default_compression_type_ = UNCOMPRESSED;
#endif
+ // Enforce aligned page size limits strictly by default.
+ g_config_value_.strict_page_size_ = true;
}
extern TSEncoding get_value_encoder(TSDataType data_type) {
@@ -104,6 +106,10 @@ void config_set_max_degree_of_index_node(uint32_t max_degree_of_index_node) {
g_config_value_.max_degree_of_index_node_ = max_degree_of_index_node;
}
+void config_set_strict_page_size(bool strict_page_size) {
+ g_config_value_.strict_page_size_ = strict_page_size;
+}
+
void set_config_value() {}
const char* s_data_type_names[8] = {"BOOLEAN", "INT32", "INT64", "FLOAT",
"DOUBLE", "TEXT", "VECTOR", "STRING"};
diff --git a/cpp/src/writer/time_chunk_writer.h b/cpp/src/writer/time_chunk_writer.h
index b5d9f489d..bf5307d53 100644
--- a/cpp/src/writer/time_chunk_writer.h
+++ b/cpp/src/writer/time_chunk_writer.h
@@ -42,7 +42,8 @@ class TimeChunkWriter {
first_page_data_(),
first_page_statistic_(nullptr),
chunk_header_(),
- num_of_pages_(0) {}
+ num_of_pages_(0),
+ enable_page_seal_if_full_(true) {}
~TimeChunkWriter() { destroy(); }
int init(const common::ColumnSchema& col_schema);
int init(const std::string& measurement_name, common::TSEncoding encoding,
@@ -57,8 +58,12 @@ class TimeChunkWriter {
if (RET_FAIL(time_page_writer_.write(timestamp))) {
return ret;
}
- if (RET_FAIL(seal_cur_page_if_full())) {
+ if (UNLIKELY(!enable_page_seal_if_full_)) {
return ret;
+ } else {
+ if (RET_FAIL(seal_cur_page_if_full())) {
+ return ret;
+ }
}
return ret;
}
@@ -68,6 +73,11 @@ class TimeChunkWriter {
Statistic* get_chunk_statistic() { return chunk_statistic_; }
FORCE_INLINE int32_t num_of_pages() const { return num_of_pages_; }
+ // Current (unsealed) page point count.
+ FORCE_INLINE uint32_t get_point_numer() const {
+ return time_page_writer_.get_point_numer();
+ }
+
int64_t estimate_max_series_mem_size();
bool hasData();
@@ -84,6 +94,12 @@ class TimeChunkWriter {
*/
int seal_current_page() { return seal_cur_page(false); }
+ // For aligned writer: allow disabling the automatic page-size/point-number
+ // check so the caller can seal pages at chosen boundaries.
+ FORCE_INLINE void set_enable_page_seal_if_full(bool enable) {
+ enable_page_seal_if_full_ = enable;
+ }
+
private:
FORCE_INLINE bool is_cur_page_full() const {
// FIXME
@@ -122,6 +138,8 @@ class TimeChunkWriter {
ChunkHeader chunk_header_;
int32_t num_of_pages_;
+ // If false, write() won't auto-seal when the current page becomes full.
+ bool enable_page_seal_if_full_;
};
} // end namespace storage
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 982c9b068..3b9d421c1 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -60,6 +60,10 @@ void set_max_degree_of_index_node(uint32_t max_degree_of_index_node) {
config_set_max_degree_of_index_node(max_degree_of_index_node);
}
+void set_strict_page_size(bool strict_page_size) {
+ config_set_strict_page_size(strict_page_size);
+}
+
TsFileWriter::TsFileWriter()
: write_file_(nullptr),
io_writer_(nullptr),
@@ -764,35 +768,217 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
data_types))) {
return ret;
}
- for (uint32_t row = 0; row < tablet.get_cur_row_size(); row++) {
- int32_t time_pages_before = time_chunk_writer->num_of_pages();
- std::vector value_pages_before(value_chunk_writers.size(), 0);
+ const uint32_t total_rows = tablet.get_cur_row_size();
+ const bool strict_page_size = common::g_config_value_.strict_page_size_;
+
+ // Decide whether we have string/blob/text columns.
+ bool has_varlen_column = false;
+ for (uint32_t i = 0; i < data_types.size(); i++) {
+ if (data_types[i] == common::STRING || data_types[i] == common::TEXT ||
+ data_types[i] == common::BLOB) {
+ has_varlen_column = true;
+ break;
+ }
+ }
+
+ // Keep writers' seal-check behavior consistent across calls.
+ time_chunk_writer->set_enable_page_seal_if_full(strict_page_size);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(
+ strict_page_size);
+ }
+ }
+
+ if (strict_page_size) {
+ // Strict mode: keep the original row-based insertion to ensure aligned
+ // pages seal together when either side becomes full.
+ for (uint32_t row = 0; row < total_rows; row++) {
+ int32_t time_pages_before = time_chunk_writer->num_of_pages();
+ std::vector value_pages_before(value_chunk_writers.size(),
+ 0);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer)) {
+ value_pages_before[c] = value_chunk_writer->num_of_pages();
+ }
+ }
+
+ if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[row]))) {
+ return ret;
+ }
+ ASSERT(value_chunk_writers.size() == tablet.get_column_count());
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ if (RET_FAIL(value_write_column(value_chunk_writer, tablet, c,
+ row, row + 1))) {
+ return ret;
+ }
+ }
+ if (RET_FAIL(maybe_seal_aligned_pages_together(
+ time_chunk_writer, value_chunk_writers, time_pages_before,
+ value_pages_before))) {
+ return ret;
+ }
+ }
+ return ret;
+ }
+
+ // Non-strict mode: switch to column-based insertion.
+ if (!has_varlen_column) {
+ // Optimization: when there is no string/blob/text column, we only need
+ // to split by point-number so that each split will trigger a page
+ // seal (and avoid the per-row page-size check).
+ const uint32_t points_per_page =
+ common::g_config_value_.page_writer_max_point_num_;
+
+ // Disable auto page sealing. We will seal pages at split boundaries.
+ time_chunk_writer->set_enable_page_seal_if_full(false);
for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
- ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
- if (!IS_NULL(value_chunk_writer)) {
- value_pages_before[c] = value_chunk_writer->num_of_pages();
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(false);
}
}
- if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[row]))) {
- return ret;
+ // Determine how many points we need to fill the current unsealed time
+ // page (it may already contain data from previous tablets).
+ uint32_t time_cur_points = time_chunk_writer->get_point_numer();
+ if (time_cur_points >= points_per_page &&
+ time_chunk_writer->has_current_page_data()) {
+ // Close the already-full page together with all aligned value
+ // pages.
+ if (RET_FAIL(time_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ if (!IS_NULL(value_chunk_writer) &&
+ value_chunk_writer->has_current_page_data()) {
+ if (RET_FAIL(value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ }
+ time_cur_points = 0;
}
+ const uint32_t first_seg_len =
+ (time_cur_points > 0 && time_cur_points < points_per_page)
+ ? (points_per_page - time_cur_points)
+ : points_per_page;
+
+ // 1) Write time in segments and seal all full segments (except the
+ // last remaining segment).
+ uint32_t seg_start = 0;
+ uint32_t seg_len = first_seg_len;
+ while (seg_start < total_rows) {
+ const uint32_t seg_end = std::min(seg_start + seg_len, total_rows);
+ if (RET_FAIL(time_write_column(time_chunk_writer, tablet, seg_start,
+ seg_end))) {
+ return ret;
+ }
+ seg_start = seg_end;
+ if (seg_start < total_rows) {
+ if (RET_FAIL(time_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ seg_len = points_per_page;
+ }
+
+ // 2) Write each value column in the same segments.
ASSERT(value_chunk_writers.size() == tablet.get_column_count());
- for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
- ValueChunkWriter* value_chunk_writer = value_chunk_writers[c];
+ for (uint32_t col = 0; col < value_chunk_writers.size(); col++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[col];
if (IS_NULL(value_chunk_writer)) {
continue;
}
- if (RET_FAIL(value_write_column(value_chunk_writer, tablet, c, row,
- row + 1))) {
- return ret;
+
+ seg_start = 0;
+ seg_len = first_seg_len;
+ while (seg_start < total_rows) {
+ const uint32_t seg_end =
+ std::min(seg_start + seg_len, total_rows);
+ if (RET_FAIL(value_write_column(value_chunk_writer, tablet, col,
+ seg_start, seg_end))) {
+ return ret;
+ }
+ seg_start = seg_end;
+ if (seg_start < total_rows) {
+ if (value_chunk_writer->has_current_page_data() &&
+ RET_FAIL(value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ seg_len = points_per_page;
}
}
- if (RET_FAIL(maybe_seal_aligned_pages_together(
- time_chunk_writer, value_chunk_writers, time_pages_before,
- value_pages_before))) {
+ return ret;
+ }
+
+ // General non-strict (may have varlen STRING/TEXT/BLOB columns):
+ // time auto-seals to provide aligned page boundaries; value writers
+ // skip auto page sealing and are sealed manually at time boundaries.
+ // Attention: since value-side auto-seal is disabled, if a varlen value
+ // page hits the memory threshold earlier, it may not seal immediately
+ // and instead will be sealed later at the recorded time-page boundaries
+ // (this may sacrifice the strict page size limit for performance).
+ time_chunk_writer->set_enable_page_seal_if_full(true);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(false);
+ }
+ }
+
+ std::vector time_page_row_ends;
+ time_page_row_ends.reserve(total_rows / 16 + 1);
+
+ // Write time and record where a time page is sealed.
+ for (uint32_t row = 0; row < total_rows; row++) {
+ const int32_t pages_before = time_chunk_writer->num_of_pages();
+ if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[row]))) {
return ret;
}
+ const int32_t pages_after = time_chunk_writer->num_of_pages();
+ if (pages_after > pages_before) {
+ const uint32_t boundary_end = row + 1;
+ if (time_page_row_ends.empty() ||
+ time_page_row_ends.back() != boundary_end) {
+ time_page_row_ends.push_back(boundary_end);
+ }
+ }
+ }
+
+ // Write values column-by-column and seal at recorded boundaries.
+ ASSERT(value_chunk_writers.size() == tablet.get_column_count());
+ for (uint32_t col = 0; col < value_chunk_writers.size(); col++) {
+ ValueChunkWriter* value_chunk_writer = value_chunk_writers[col];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ uint32_t seg_start = 0;
+ for (uint32_t boundary_end : time_page_row_ends) {
+ if (boundary_end <= seg_start) {
+ continue;
+ }
+ if (RET_FAIL(value_write_column(value_chunk_writer, tablet, col,
+ seg_start, boundary_end))) {
+ return ret;
+ }
+ if (value_chunk_writer->has_current_page_data() &&
+ RET_FAIL(value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ seg_start = boundary_end;
+ }
+ if (seg_start < total_rows) {
+ if (RET_FAIL(value_write_column(value_chunk_writer, tablet, col,
+ seg_start, total_rows))) {
+ return ret;
+ }
+ }
}
return ret;
}
@@ -874,35 +1060,235 @@ int TsFileWriter::write_table(Tablet& tablet) {
value_chunk_writers))) {
return ret;
}
- // Row-by-row write so that when time page seals (e.g. by memory
- // threshold), we can seal all value pages together (Java
- // semantics).
- for (int i = start_idx; i < end_idx; i++) {
- int32_t time_pages_before = time_chunk_writer->num_of_pages();
- if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[i]))) {
- return ret;
+
+ const bool strict_page_size =
+ common::g_config_value_.strict_page_size_;
+
+ std::vector field_columns;
+ field_columns.reserve(tablet.get_column_count());
+ for (uint32_t col = 0; col < tablet.get_column_count(); ++col) {
+ if (tablet.column_categories_[col] ==
+ common::ColumnCategory::FIELD) {
+ field_columns.push_back(col);
+ }
+ }
+ ASSERT(field_columns.size() == value_chunk_writers.size());
+
+ const bool has_varlen_field_column = [&]() {
+ for (uint32_t i = 0; i < field_columns.size(); i++) {
+ const common::TSDataType t =
+ tablet.schema_vec_->at(field_columns[i]).data_type_;
+ if (t == common::STRING || t == common::TEXT ||
+ t == common::BLOB) {
+ return true;
+ }
+ }
+ return false;
+ }();
+
+ // Keep writers' seal-check behavior consistent across calls.
+ time_chunk_writer->set_enable_page_seal_if_full(strict_page_size);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(
+ strict_page_size);
}
- uint32_t field_col_count = 0;
- for (uint32_t col = 0; col < tablet.get_column_count(); ++col) {
- if (tablet.column_categories_[col] ==
- common::ColumnCategory::FIELD) {
+ }
+
+ if (strict_page_size) {
+ // Strict: row-based insertion and force aligned page sealing
+ // when either time or any value page becomes full.
+ for (int i = start_idx; i < end_idx; i++) {
+ int32_t time_pages_before =
+ time_chunk_writer->num_of_pages();
+ std::vector value_pages_before(
+ value_chunk_writers.size(), 0);
+ for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
+ if (!IS_NULL(value_chunk_writers[k])) {
+ value_pages_before[k] =
+ value_chunk_writers[k]->num_of_pages();
+ }
+ }
+
+ if (RET_FAIL(
+ time_chunk_writer->write(tablet.timestamps_[i]))) {
+ return ret;
+ }
+
+ for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
ValueChunkWriter* value_chunk_writer =
- value_chunk_writers[field_col_count];
- if (!IS_NULL(value_chunk_writer) &&
- RET_FAIL(value_write_column(
- value_chunk_writer, tablet, col, i, i + 1))) {
+ value_chunk_writers[k];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ const uint32_t tablet_col_idx = field_columns[k];
+ if (RET_FAIL(value_write_column(value_chunk_writer,
+ tablet, tablet_col_idx,
+ i, i + 1))) {
return ret;
}
- field_col_count++;
+ }
+
+ if (RET_FAIL(maybe_seal_aligned_pages_together(
+ time_chunk_writer, value_chunk_writers,
+ time_pages_before, value_pages_before))) {
+ return ret;
}
}
- int32_t time_pages_after = time_chunk_writer->num_of_pages();
- if (time_pages_after > time_pages_before) {
- for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
- if (!IS_NULL(value_chunk_writers[k]) &&
- value_chunk_writers[k]->get_point_numer() > 0 &&
- RET_FAIL(
- value_chunk_writers[k]->seal_current_page())) {
+ } else if (!has_varlen_field_column) {
+ // Optimization: no string/blob/text columns, so we can
+ // segment by point-number and seal pages at those boundaries
+ // in column-based order.
+ const uint32_t points_per_page =
+ common::g_config_value_.page_writer_max_point_num_;
+
+ time_chunk_writer->set_enable_page_seal_if_full(false);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(
+ false);
+ }
+ }
+
+ // Fill the already-unsealed time page first.
+ uint32_t time_cur_points = time_chunk_writer->get_point_numer();
+ if (time_cur_points >= points_per_page &&
+ time_chunk_writer->has_current_page_data()) {
+ if (RET_FAIL(time_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c]) &&
+ value_chunk_writers[c]->has_current_page_data()) {
+ if (RET_FAIL(value_chunk_writers[c]
+ ->seal_current_page())) {
+ return ret;
+ }
+ }
+ }
+ time_cur_points = 0;
+ }
+
+ const uint32_t first_seg_len =
+ (time_cur_points > 0 && time_cur_points < points_per_page)
+ ? (points_per_page - time_cur_points)
+ : points_per_page;
+
+ // 1) Write time in segments (seal all full segments).
+ uint32_t seg_start = static_cast(start_idx);
+ uint32_t seg_len = first_seg_len;
+ while (static_cast(seg_start) < end_idx) {
+ const uint32_t seg_end = std::min(
+ seg_start + seg_len, static_cast(end_idx));
+ if (RET_FAIL(time_write_column(time_chunk_writer, tablet,
+ seg_start, seg_end))) {
+ return ret;
+ }
+ seg_start = seg_end;
+ if (static_cast(seg_start) < end_idx) {
+ if (RET_FAIL(time_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ seg_len = points_per_page;
+ }
+
+ // 2) Write each value column (same segments).
+ for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
+ ValueChunkWriter* value_chunk_writer =
+ value_chunk_writers[k];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ seg_start = static_cast(start_idx);
+ seg_len = first_seg_len;
+ while (static_cast(seg_start) < end_idx) {
+ const uint32_t seg_end =
+ std::min(seg_start + seg_len,
+ static_cast(end_idx));
+ if (RET_FAIL(value_write_column(
+ value_chunk_writer, tablet, field_columns[k],
+ seg_start, seg_end))) {
+ return ret;
+ }
+ seg_start = seg_end;
+ if (static_cast(seg_start) < end_idx) {
+ if (value_chunk_writer->has_current_page_data() &&
+ RET_FAIL(
+ value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ }
+ seg_len = points_per_page;
+ }
+ }
+ } else {
+ // General non-strict (may have varlen STRING/TEXT/BLOB
+ // columns): time auto-seals to provide aligned page boundaries;
+ // value writers skip auto page sealing and are sealed manually
+ // at recorded time-page boundaries. Attention: since value-side
+ // auto-seal is disabled, if a varlen value page hits the memory
+ // threshold earlier, it may not seal immediately and will be
+ // sealed later at the time-page boundaries (non-strict
+ // sacrifices the strict page size/memory limit for
+ // performance).
+ time_chunk_writer->set_enable_page_seal_if_full(true);
+ for (uint32_t c = 0; c < value_chunk_writers.size(); c++) {
+ if (!IS_NULL(value_chunk_writers[c])) {
+ value_chunk_writers[c]->set_enable_page_seal_if_full(
+ false);
+ }
+ }
+
+ std::vector time_page_row_ends;
+ time_page_row_ends.reserve((end_idx - start_idx) / 16 + 1);
+ for (uint32_t r = static_cast(start_idx);
+ r < static_cast(end_idx); r++) {
+ const int32_t pages_before =
+ time_chunk_writer->num_of_pages();
+ if (RET_FAIL(
+ time_chunk_writer->write(tablet.timestamps_[r]))) {
+ return ret;
+ }
+ const int32_t pages_after =
+ time_chunk_writer->num_of_pages();
+ if (pages_after > pages_before) {
+ const uint32_t boundary_end = r + 1;
+ if (time_page_row_ends.empty() ||
+ time_page_row_ends.back() != boundary_end) {
+ time_page_row_ends.push_back(boundary_end);
+ }
+ }
+ }
+
+ // Write values column-by-column and seal at recorded time
+ // boundaries.
+ for (uint32_t k = 0; k < value_chunk_writers.size(); k++) {
+ ValueChunkWriter* value_chunk_writer =
+ value_chunk_writers[k];
+ if (IS_NULL(value_chunk_writer)) {
+ continue;
+ }
+ uint32_t seg_start = static_cast(start_idx);
+ for (uint32_t boundary_end : time_page_row_ends) {
+ if (boundary_end <= seg_start) {
+ continue;
+ }
+ if (RET_FAIL(value_write_column(
+ value_chunk_writer, tablet, field_columns[k],
+ seg_start, boundary_end))) {
+ return ret;
+ }
+ if (value_chunk_writer->has_current_page_data() &&
+ RET_FAIL(value_chunk_writer->seal_current_page())) {
+ return ret;
+ }
+ seg_start = boundary_end;
+ }
+ if (seg_start < static_cast(end_idx)) {
+ if (RET_FAIL(value_write_column(
+ value_chunk_writer, tablet, field_columns[k],
+ seg_start, static_cast(end_idx)))) {
return ret;
}
}
diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h
index 75f88d37b..cacf5b2ea 100644
--- a/cpp/src/writer/tsfile_writer.h
+++ b/cpp/src/writer/tsfile_writer.h
@@ -45,6 +45,7 @@ extern int libtsfile_init();
extern void libtsfile_destroy();
extern void set_page_max_point_count(uint32_t page_max_ponint_count);
extern void set_max_degree_of_index_node(uint32_t max_degree_of_index_node);
+extern void set_strict_page_size(bool strict_page_size);
class TsFileWriter {
public:
diff --git a/cpp/src/writer/value_chunk_writer.h b/cpp/src/writer/value_chunk_writer.h
index 6ec54022c..88c25113e 100644
--- a/cpp/src/writer/value_chunk_writer.h
+++ b/cpp/src/writer/value_chunk_writer.h
@@ -53,7 +53,8 @@ class ValueChunkWriter {
first_page_data_(),
first_page_statistic_(nullptr),
chunk_header_(),
- num_of_pages_(0) {}
+ num_of_pages_(0),
+ enable_page_seal_if_full_(true) {}
~ValueChunkWriter() { destroy(); }
int init(const common::ColumnSchema& col_schema);
int init(const std::string& measurement_name, common::TSDataType data_type,
@@ -135,6 +136,12 @@ class ValueChunkWriter {
*/
int seal_current_page() { return seal_cur_page(false); }
+ // For aligned writer: allow disabling the automatic page-size/point-number
+ // check so the caller can seal pages at chosen boundaries.
+ FORCE_INLINE void set_enable_page_seal_if_full(bool enable) {
+ enable_page_seal_if_full_ = enable;
+ }
+
private:
FORCE_INLINE bool is_cur_page_full() const {
// FIXME
@@ -144,6 +151,9 @@ class ValueChunkWriter {
common::g_config_value_.page_writer_max_memory_bytes_);
}
FORCE_INLINE int seal_cur_page_if_full() {
+ if (UNLIKELY(!enable_page_seal_if_full_)) {
+ return common::E_OK;
+ }
if (UNLIKELY(is_cur_page_full())) {
return seal_cur_page(false);
}
@@ -173,6 +183,8 @@ class ValueChunkWriter {
ChunkHeader chunk_header_;
int32_t num_of_pages_;
+ // If false, write() won't auto-seal when the current page becomes full.
+ bool enable_page_seal_if_full_;
};
} // end namespace storage
From 633b9e1fad73c4150414e4f26c35aa0fc789214e Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Thu, 26 Mar 2026 16:47:08 +0800
Subject: [PATCH 12/13] fix QueryByRowFasterThanManualNext tolerance
---
cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc | 4 ++--
cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc b/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc
index 13a0257d3..8003326e9 100644
--- a/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc
+++ b/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc
@@ -651,7 +651,7 @@ TEST_F(TableQueryByRowTest, DenseSingleDeviceSsiLevelPushdown) {
// Pushdown is faster than full query + manual next: queryByRow(offset, limit)
// skips at device/SSI/Chunk level; old query then manual next decodes every
-// row. Timing tolerance 5% to allow measurement noise.
+// row. Timing tolerance 20% to allow measurement noise.
TEST_F(TableQueryByRowTest, QueryByRowFasterThanManualNext) {
const int num_rows = 8000;
const int offset = 3000;
@@ -659,7 +659,7 @@ TEST_F(TableQueryByRowTest, QueryByRowFasterThanManualNext) {
write_single_device_file(num_rows);
const int num_iters = 5;
- const double tolerance = 0.1; // 10% tolerance to allow for timing noise
+ const double tolerance = 0.2;
auto run_query_by_row = [this, offset, limit]() {
TsFileReader reader;
diff --git a/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc b/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc
index 56f8c113a..1643303df 100644
--- a/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc
+++ b/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc
@@ -1102,7 +1102,7 @@ TEST_F(TreeQueryByRowTest, MultiPath_TimeHint_SkipsStaleChunk_WithOffset) {
// Pushdown is faster than full query + manual next: queryByRow(offset, limit)
// skips at Chunk/Page level; old query then manual next decodes every row.
-// Timing tolerance 5% to allow measurement noise.
+// Timing tolerance 20% to allow measurement noise.
TEST_F(TreeQueryByRowTest, QueryByRowFasterThanManualNext) {
std::vector devices = {"d1"};
std::vector measurements = {"s1"};
@@ -1112,7 +1112,7 @@ TEST_F(TreeQueryByRowTest, QueryByRowFasterThanManualNext) {
write_test_file(devices, measurements, num_rows);
const int num_iters = 5;
- const double tolerance = 0.05;
+ const double tolerance = 0.2;
auto run_query_by_row = [this, &devices, &measurements, offset, limit]() {
TsFileTreeReader reader;
From 9f9c864e486e654258d17e96a6fa61e1a5ec753f Mon Sep 17 00:00:00 2001
From: 761417898 <761417898@qq.com>
Date: Fri, 27 Mar 2026 11:19:42 +0800
Subject: [PATCH 13/13] fix time_page_row_ends.reserve
---
cpp/src/writer/tsfile_writer.cc | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index f8a823eb1..786325db5 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -1021,7 +1021,9 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
}
std::vector time_page_row_ends;
- time_page_row_ends.reserve(total_rows / 16 + 1);
+ const uint32_t page_max_points = std::max(
+ 1, common::g_config_value_.page_writer_max_point_num_);
+ time_page_row_ends.reserve(total_rows / page_max_points + 1);
// Write time and record where a time page is sealed.
for (uint32_t row = 0; row < total_rows; row++) {
@@ -1329,7 +1331,11 @@ int TsFileWriter::write_table(Tablet& tablet) {
}
std::vector time_page_row_ends;
- time_page_row_ends.reserve((end_idx - start_idx) / 16 + 1);
+ const uint32_t page_max_points = std::max(
+ 1, common::g_config_value_.page_writer_max_point_num_);
+ const uint32_t batch_rows =
+ static_cast(end_idx - start_idx);
+ time_page_row_ends.reserve(batch_rows / page_max_points + 1);
for (uint32_t r = static_cast(start_idx);
r < static_cast(end_idx); r++) {
const int32_t pages_before =