diff --git a/.devcontainer/Dockerfile.template b/.devcontainer/Dockerfile.template index ebc256e6..00f1ee15 100644 --- a/.devcontainer/Dockerfile.template +++ b/.devcontainer/Dockerfile.template @@ -17,12 +17,35 @@ # Adapted from Apache Iceberg C++ # https://github.com/apache/iceberg-cpp/blob/v0.2.0/.devcontainer/Dockerfile.template - +# # This Dockerfile is used to build a development container for Paimon C++. -# It is based on the Ubuntu image and installs necessary dependencies. +# Base: Ubuntu 24.04. Rust toolchain is installed via Dev Container +# Feature `ghcr.io/devcontainers/features/rust:1` (see devcontainer.json), +# so it does NOT appear in this Dockerfile. FROM ubuntu:24.04 +# Optional apt mirror. Defaults to the upstream Ubuntu mirrors so builds work +# everywhere; pass --build-arg APT_MIRROR= (e.g. http://mirrors.aliyun.com) +# to use a regional mirror for faster downloads inside mainland China. +ARG APT_MIRROR= +RUN if [ -n "${APT_MIRROR}" ]; then \ + sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu|${APT_MIRROR}/ubuntu|g" \ + -e "s|http://security.ubuntu.com/ubuntu|${APT_MIRROR}/ubuntu|g" \ + -e "s|http://ports.ubuntu.com/ubuntu-ports|${APT_MIRROR}/ubuntu-ports|g" \ + /etc/apt/sources.list.d/ubuntu.sources; \ + fi + +# Optional rustup mirror for the Dev Container Feature +# `ghcr.io/devcontainers/features/rust:1` (and any later `rustup` calls). +# Defaults to the upstream static.rust-lang.org; pass +# --build-arg RUST_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static for a +# China-friendly CDN. Set as ENV so it is inherited by every subsequent layer. +ARG RUST_DIST_SERVER=https://static.rust-lang.org +ENV RUSTUP_DIST_SERVER=${RUST_DIST_SERVER} \ + RUSTUP_UPDATE_ROOT=${RUST_DIST_SERVER}/rustup + # Install necessary packages RUN apt update && \ apt install -y \ @@ -48,6 +71,16 @@ RUN apt update && \ vim \ wget \ sudo \ + # ---- additions for tantivy-fts migration (Rust + Sanitizer + LLVM) ---- + clang \ + clang-format \ + clang-tidy \ + lld \ + llvm \ + libclang-rt-dev \ + gdb \ + lldb \ + valgrind \ && rm -rf /var/lib/apt/lists/* # Add a user for development diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile new file mode 100644 index 00000000..9a25b5e8 --- /dev/null +++ b/.devcontainer/centos7/Dockerfile @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# CentOS 7 cross-build verification image for paimon-cpp + tantivy-fts. +# +# Purpose: +# Prove the tantivy-fts stack builds on the OLDEST reasonable Linux target +# (glibc 2.17, EOL 2024-06-30). The default Ubuntu 24.04 dev container +# proves nothing about glibc compatibility; this image does. +# +# Build: +# docker build -t paimon-cpp-centos7:latest -f .devcontainer/centos7/Dockerfile . +# +# Run: +# docker run -d --name paimon-centos7 \ +# --privileged \ +# -v "$(pwd):/workspaces/paimon-cpp" \ +# paimon-cpp-centos7:latest sleep infinity +# docker exec -it paimon-centos7 bash -l +# +# Inside the container: +# scl enable devtoolset-11 rh-python38 -- bash # activate modern gcc + python +# source /opt/paimon-env.sh # PATH for rust, cmake +# cd /workspaces/paimon-cpp +# +# Run ./.devcontainer/centos7/run.sh smoke from the host for the full check. + +# ---------- Base ---------- +# CentOS 7 reached EOL 2024-06-30; its default mirrorlist.centos.org is down. +# Pin to vault.centos.org (Red Hat's archived location) via the `linuxserver/centos` +# vault image to avoid retired-mirror failures on `yum install`. +# +# Base image: we pull from quay.io (CentOS community's canonical registry post +# Docker Hub deprecation). Override with CENTOS7_IMAGE build arg when behind a +# firewall that can't reach quay.io (e.g. registry.aliyuncs.com/library/centos:7). +ARG CENTOS7_IMAGE=quay.io/centos/centos:centos7 +FROM ${CENTOS7_IMAGE} + +# Repoint yum at aliyun's CentOS 7 vault mirror — vault.centos.org itself +# works but is slow/blocked from many CN networks; the aliyun mirror is a +# complete rsync and reliably fast. We overwrite CentOS-Base.repo rather +# than sed-patch it so the result is deterministic regardless of what the +# upstream image ships. fastestmirror plugin is disabled because its ping +# probes against the retired mirror list add ~60s to every `yum install`. +RUN echo -e '[base]\n\ +name=CentOS-7 - Base - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/os/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[updates]\n\ +name=CentOS-7 - Updates - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/updates/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[extras]\n\ +name=CentOS-7 - Extras - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/extras/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[centosplus]\n\ +name=CentOS-7 - Plus - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/centosplus/$basearch/\n\ +gpgcheck=0\n\ +enabled=0\n' > /etc/yum.repos.d/CentOS-Base.repo \ + && rm -f /etc/yum.repos.d/CentOS-CR.repo \ + /etc/yum.repos.d/CentOS-Debuginfo.repo \ + /etc/yum.repos.d/CentOS-Media.repo \ + /etc/yum.repos.d/CentOS-Sources.repo \ + /etc/yum.repos.d/CentOS-Vault.repo \ + /etc/yum.repos.d/CentOS-fasttrack.repo \ + /etc/yum.repos.d/CentOS-x86_64-kernel.repo \ + && if [ -f /etc/yum/pluginconf.d/fastestmirror.conf ]; then \ + sed -i 's/^enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf; \ + fi \ + && yum clean all \ + && yum makecache + +# ---------- Base toolchain ---------- +# EPEL provides ninja-build and a newer python3 than the base 3.6. +# SCL (Software Collections) provides devtoolset-11 (gcc 11) and rh-python38 +# without overriding the system gcc/python. CentOS 7's default gcc 4.8 is +# too old for C++17/20 used by lucene++ and our tantivy wrapper. +# +# Same story as CentOS-Base.repo: both epel + SCL default to mirrorlist +# endpoints that are effectively dead; overwrite with aliyun URLs that we +# know respond. +RUN yum install -y epel-release centos-release-scl \ + && echo -e '[epel]\n\ +name=Extra Packages for Enterprise Linux 7 - aliyun\n\ +baseurl=https://mirrors.aliyun.com/epel/7/$basearch\n\ +gpgcheck=0\n\ +enabled=1\n' > /etc/yum.repos.d/epel.repo \ + && rm -f /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel.repo.rpmnew \ + && rm -f /etc/yum.repos.d/CentOS-SCLo-*.repo \ + /etc/yum.repos.d/CentOS-SCLo-*.repo.rpmnew \ + && echo -e '[centos-sclo-rh]\n\ +name=CentOS-7 - SCLo rh - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/sclo/$basearch/rh/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[centos-sclo-sclo]\n\ +name=CentOS-7 - SCLo sclo - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/sclo/$basearch/sclo/\n\ +gpgcheck=0\n\ +enabled=1\n' > /etc/yum.repos.d/CentOS-SCLo-scl.repo \ + && yum clean all && yum makecache \ + && yum install -y \ + devtoolset-11-gcc \ + devtoolset-11-gcc-c++ \ + devtoolset-11-binutils \ + devtoolset-11-libasan-devel \ + devtoolset-11-libubsan-devel \ + rh-python38 \ + rh-python38-python-pip \ + git \ + ninja-build \ + make \ + patch \ + curl \ + wget \ + unzip \ + which \ + file \ + sudo \ + openssl-devel \ + zlib-devel \ + libffi-devel \ + bzip2-devel \ + xz-devel \ + perl-IPC-Cmd \ + && yum clean all + +# Enable the SCL collections for all subsequent shells (including RUN). +ENV BASH_ENV=/etc/profile.d/scl-enable.sh +SHELL ["/bin/bash", "-c"] +RUN printf '%s\n' \ + 'source scl_source enable devtoolset-11' \ + 'source scl_source enable rh-python38' \ + > /etc/profile.d/scl-enable.sh \ + && chmod +x /etc/profile.d/scl-enable.sh + +# ---------- CMake (must be >= 3.22 for Corrosion) ---------- +# CentOS 7's cmake package is 2.8.12; EPEL cmake3 is 3.17 — still too old. +# Install via pip in the rh-python38 SCL so we get a modern CMake without +# touching the system /usr/bin. Point pip at aliyun's pypi mirror: default +# pypi.org is 10-30s per request from CN, aliyun responds in <1s. +ENV PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ \ + PIP_TRUSTED_HOST=mirrors.aliyun.com +RUN source /etc/profile.d/scl-enable.sh \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install 'cmake==3.28.*' ninja + +# ---------- Rust toolchain ---------- +# Install rustup as root into /opt/rust so all users share the same toolchain. +# Use the USTC mirror to keep downloads fast in CN; the CI runner version of +# this is mirrored in ci/scripts/setup_rust.sh. +ENV RUSTUP_HOME=/opt/rust/rustup \ + CARGO_HOME=/opt/rust/cargo \ + RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static \ + RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup +# In-container network for Docker Desktop builds is unreliable through many +# CN mirrors (observed: curl 7.29 on CentOS 7 + rsproxy.cn HTTP/2 path ⇒ +# partial-read truncations; USTC ⇒ 5xx; rustup sh installer ⇒ 403 from +# legacy cipher). The most reliable fix is to sidestep the issue entirely: +# pre-download rustup-init on the host (where network is solid) and COPY it +# into the image. See .devcontainer/centos7/run.sh for the prefetch step. +COPY .devcontainer/centos7/rustup-init.bin /tmp/rustup-init +RUN chmod +x /tmp/rustup-init \ + && /tmp/rustup-init -y --default-toolchain stable --profile minimal --no-modify-path \ + && rm -f /tmp/rustup-init \ + && mkdir -p $CARGO_HOME \ + && echo -e '[source.crates-io]\n\ +replace-with = "rsproxy-sparse"\n\ +\n\ +[source.rsproxy]\n\ +registry = "https://rsproxy.cn/crates.io-index"\n\ +\n\ +[source.rsproxy-sparse]\n\ +registry = "sparse+https://rsproxy.cn/index/"\n\ +\n\ +[registries.rsproxy]\n\ +index = "https://rsproxy.cn/crates.io-index"\n\ +\n\ +[net]\n\ +git-fetch-with-cli = true\n' > $CARGO_HOME/config.toml \ + && $CARGO_HOME/bin/cargo install cbindgen --version 0.29.2 --locked \ + && chmod -R a+rwx /opt/rust \ + && $CARGO_HOME/bin/rustc --version \ + && $CARGO_HOME/bin/cargo --version \ + && $CARGO_HOME/bin/cbindgen --version + +# ---------- Environment file consumed by every shell ---------- +# Sets PATH for rust / cmake / cargo so `docker exec paimon-centos7 bash -l` +# and interactive sessions have the toolchain on $PATH. +RUN printf '%s\n' \ + 'export PATH=/opt/rust/cargo/bin:$PATH' \ + '# cmake + ninja live under the rh-python38 SCL; path prefix differs by arch.' \ + '# `command -v cmake` confirms which one is in use.' \ + > /opt/paimon-env.sh \ + && chmod +x /opt/paimon-env.sh \ + && printf '%s\n' 'source /opt/paimon-env.sh' >> /etc/profile.d/scl-enable.sh + +# ---------- Non-root user ---------- +# Build as `paimon` (uid 1000) so LFS objects under the mount stay owned by +# your host user, matching the main Ubuntu dev container. +RUN useradd -m -u 1000 -s /bin/bash paimon \ + && echo 'paimon ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/paimon + +USER paimon +WORKDIR /workspaces/paimon-cpp + +# Sanity check surfaces the tool versions in `docker run ... paimon-cpp-centos7 --version`. +CMD ["bash", "-lc", "\ + echo '--- CentOS 7 cross-build image sanity check ---'; \ + cat /etc/centos-release; \ + echo '--- glibc ---'; ldd --version | head -1; \ + echo '--- gcc ---'; gcc --version | head -1; \ + echo '--- cmake ---'; cmake --version | head -1; \ + echo '--- ninja ---'; ninja --version; \ + echo '--- rust ---'; rustc --version; \ + echo '--- cargo ---'; cargo --version; \ + echo '--- cbindgen ---'; cbindgen --version; \ + echo 'Ready. From the host repository, run ./.devcontainer/centos7/run.sh smoke'"] diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh new file mode 100755 index 00000000..2f494f27 --- /dev/null +++ b/.devcontainer/centos7/run.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# One-shot helper to build + launch + smoke-test the CentOS 7 verification +# container. Run from the paimon-cpp repo root. +# +# Usage: +# ./.devcontainer/centos7/run.sh build # build image only +# ./.devcontainer/centos7/run.sh up # start container (detached) +# ./.devcontainer/centos7/run.sh shell # exec into it +# ./.devcontainer/centos7/run.sh smoke # run the smoke suite inside +# ./.devcontainer/centos7/run.sh down # stop + remove + +set -euo pipefail + +IMAGE=paimon-cpp-centos7:latest +CONTAINER=paimon-centos7 + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +cmd=${1:-help} + +case "${cmd}" in + build) + # Prefetch rustup-init on the host. In-container network from Docker + # Desktop builds is unreliable for CN mirrors (TLS/HTTP2 issues with + # old curl/wget on CentOS 7), but host curl works. The image copies + # this blob in. Override mirror with RUSTUP_INIT_URL=... if needed. + rustup_init="${here}/rustup-init.bin" + rustup_url="${RUSTUP_INIT_URL:-https://mirrors.ustc.edu.cn/rust-static/rustup/dist/x86_64-unknown-linux-gnu/rustup-init}" + if [ ! -s "${rustup_init}" ]; then + echo "==> Prefetching rustup-init from ${rustup_url}" + curl --proto '=https' --tlsv1.2 -sSfL --retry 5 --retry-delay 5 \ + -o "${rustup_init}" "${rustup_url}" + fi + # Override base image with CENTOS7_IMAGE=... if quay.io is unreachable. + # Common fallbacks you may need to docker-pull into local cache first: + # CENTOS7_IMAGE=quay.io/centos/centos:centos7 (default) + # CENTOS7_IMAGE=registry.aliyuncs.com/library/centos:7 + if [ -n "${CENTOS7_IMAGE:-}" ]; then + docker build -t "${IMAGE}" -f "${here}/Dockerfile" \ + --build-arg "CENTOS7_IMAGE=${CENTOS7_IMAGE}" "${repo}" + else + docker build -t "${IMAGE}" -f "${here}/Dockerfile" "${repo}" + fi + ;; + up) + docker rm -f "${CONTAINER}" 2>/dev/null || true + docker run -d \ + --name "${CONTAINER}" \ + --privileged \ + -v "${repo}:/workspaces/paimon-cpp" \ + -v "paimon-centos7-cargo-registry:/opt/rust/cargo/registry" \ + -v "paimon-centos7-build:/workspaces/paimon-cpp/build-centos7" \ + "${IMAGE}" sleep infinity + # Named volumes mount as root-owned; `paimon` user (uid 1000) needs + # write access to build-centos7 and the cargo registry cache. + docker exec --user root "${CONTAINER}" bash -c ' + chown -R paimon:paimon /workspaces/paimon-cpp/build-centos7 \ + /opt/rust/cargo/registry + ' + echo "Container started. \`${0} shell\` to enter." + ;; + shell) + docker exec -it "${CONTAINER}" bash -l + ;; + smoke) + # Ensure container is up first; no-op if already running. + if ! docker ps --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + echo "Container ${CONTAINER} not running; starting it." + "$0" up + fi + # Set two environment variables for Rosetta 2 (Apple Silicon) compatibility: + # MALLOC_CHECK_=0 disables glibc 2.17 extra malloc integrity checks + # that fire false positives under Rosetta's x86_64 emulation. + # ARROW_USER_SIMD_LEVEL=SSE4_2 keeps arrow runtime-dispatched kernels + # on SSE4.2 only (Rosetta does not support AVX2/BMI2/AVX-512). + # Both are no-ops on real x86_64 CentOS 7 hardware. + # Use a distinct build dir inside the container so it does not clash + # with the Ubuntu dev container's build/ dir on the same volume. + docker exec \ + -e "MALLOC_CHECK_=0" \ + -e "ARROW_USER_SIMD_LEVEL=SSE4_2" \ + "${CONTAINER}" bash -lc ' + set -eux + cd /workspaces/paimon-cpp + cmake -S . -B build-centos7 \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DPAIMON_BUILD_TESTS=ON \ + -DPAIMON_ENABLE_FSLIB=OFF \ + -DPAIMON_ENABLE_LUMINA=OFF \ + -DPAIMON_ENABLE_JINDO=OFF \ + -DPAIMON_ENABLE_LUCENE=ON \ + -DPAIMON_ENABLE_TANTIVY=ON \ + -DPAIMON_ENABLE_ORC=ON \ + -DPAIMON_ENABLE_AVRO=ON + cmake --build build-centos7 -j "$(nproc)" + ctest --test-dir build-centos7 \ + -R "paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-.*-test" \ + --output-on-failure + ' + ;; + down) + docker rm -f "${CONTAINER}" 2>/dev/null || true + ;; + help|*) + sed -n "2,20p" "$0" + ;; +esac diff --git a/.devcontainer/devcontainer.json.template b/.devcontainer/devcontainer.json.template index d4f7c347..f5a6c742 100644 --- a/.devcontainer/devcontainer.json.template +++ b/.devcontainer/devcontainer.json.template @@ -20,6 +20,10 @@ // Adapted from Apache Iceberg C++ // https://github.com/apache/iceberg-cpp/blob/v0.2.0/.devcontainer/devcontainer.json.template +// Default Paimon C++ Dev Container. +// On Apple Silicon hosts this runs as native aarch64 Linux (fast). +// For x86_64 verification, use the variant under .devcontainer/x86_64/. + { "name": "Paimon CPP Dev Container", "build": { @@ -34,16 +38,36 @@ "seccomp=unconfined", "--privileged" ], + "features": { + "ghcr.io/devcontainers/features/rust:1": { + "version": "stable", + "profile": "default" + } + }, "mounts": [ - "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly" + "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", + "source=paimon-cargo-registry,target=/home/paimon/.cargo/registry,type=volume", + "source=paimon-cargo-git,target=/home/paimon/.cargo/git,type=volume", + "source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume", + "source=paimon-build,target=${containerWorkspaceFolder}/build,type=volume", + "source=paimon-ccache,target=/home/paimon/.ccache,type=volume" ], + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", "customizations": { "vscode": { "extensions": [ - "eamodio.gitlens" + "eamodio.gitlens", + "rust-lang.rust-analyzer", + "vadimcn.vscode-lldb", + "llvm-vs-code-extensions.vscode-clangd", + "ms-vscode.cmake-tools", + "twxs.cmake" ], "settings": { - "editor.formatOnSave": true + "editor.formatOnSave": true, + "rust-analyzer.linkedProjects": [ + "crates/tantivy_ffi/Cargo.toml" + ] } } } diff --git a/.devcontainer/x86_64/devcontainer.json.template b/.devcontainer/x86_64/devcontainer.json.template new file mode 100644 index 00000000..00763314 --- /dev/null +++ b/.devcontainer/x86_64/devcontainer.json.template @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// x86_64 variant of the Paimon CPP Dev Container. +// On Apple Silicon hosts this runs under QEMU emulation (5-10x slower). +// Use it only for cross-architecture verification, not daily development. +// +// Reuses the same Dockerfile as the default container; only the platform differs. +// +// Uses dedicated named volumes (suffix `-amd64`) so build/cargo cache do not +// collide with the native aarch64 container. + +{ + "name": "Paimon CPP Dev Container (x86_64 via QEMU)", + "build": { + "dockerfile": "../Dockerfile", + "options": [ + "--platform=linux/amd64" + ] + }, + "runArgs": [ + "--platform=linux/amd64", + "--ulimit=core=-1", + "--cap-add=SYS_ADMIN", + "--cap-add=SYS_PTRACE", + "--cap-add=PERFMON", + "--security-opt", + "seccomp=unconfined", + "--privileged" + ], + "features": { + "ghcr.io/devcontainers/features/rust:1": { + "version": "stable", + "profile": "default" + } + }, + "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", + "source=paimon-cargo-registry-amd64,target=/home/paimon/.cargo/registry,type=volume", + "source=paimon-cargo-git-amd64,target=/home/paimon/.cargo/git,type=volume", + "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume", + "source=paimon-build-amd64,target=${containerWorkspaceFolder}/build,type=volume", + "source=paimon-ccache-amd64,target=/home/paimon/.ccache,type=volume" + ], + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "rust-lang.rust-analyzer", + "vadimcn.vscode-lldb", + "llvm-vs-code-extensions.vscode-clangd", + "ms-vscode.cmake-tools" + ], + "settings": { + "editor.formatOnSave": true + } + } + } +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..e49bec47 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +CLAUDE.md export-ignore diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1320da73..50d87f7f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ - + ### Purpose diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_and_test.yaml similarity index 50% rename from .github/workflows/build_release.yaml rename to .github/workflows/build_and_test.yaml index 0bed7ced..a015342b 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_and_test.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Build Release +name: Build and Test on: push: @@ -33,45 +33,60 @@ permissions: contents: read jobs: - clang-release: + build-and-test: + name: ${{ matrix.name }} runs-on: ubuntu-24.04 timeout-minutes: 120 strategy: fail-fast: false + matrix: + include: + - name: gcc-release + cc: gcc-14 + cxx: g++-14 + build_args: --build_type Release + - name: clang-release + build_args: --build_type Release + - name: gcc-debug + cc: gcc-14 + cxx: g++-14 + - name: clang-debug + # fetch_depth: '0' # fetch the PR target branch history for clang-tidy + # build_args: >- + # --check_clang_tidy + # --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}" + - name: asan-ubsan + build_args: --enable_asan --enable_ubsan + - name: tsan + skip_rust: true + build_args: --enable_tsan steps: - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache with: - cache-key-prefix: ccache-clang-release - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false false Release - - name: Show ccache statistics - if: always() - run: ccache -s - gcc-release: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + lfs: true + fetch-depth: ${{ matrix.fetch_depth || '1' }} - name: Setup ccache uses: ./.github/actions/setup-ccache with: - cache-key-prefix: ccache-gcc-release + cache-key-prefix: ccache-${{ matrix.name }} + - name: Install Rust toolchain (tantivy-fts) + if: ${{ !matrix.skip_rust }} + shell: bash + run: ci/scripts/setup_rust.sh + - name: Install HTTP and TLS development dependencies + run: | + sudo apt-get update + sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Build Paimon shell: bash env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) false false Release + CC: ${{ matrix.cc || 'clang' }} + CXX: ${{ matrix.cxx || 'clang++' }} + run: >- + ci/scripts/build_paimon.sh + --source_dir "$(pwd)" + ${{ matrix.build_args }} - name: Show ccache statistics if: always() run: ccache -s diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml deleted file mode 100644 index ea30689c..00000000 --- a/.github/workflows/clang_test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Clang Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - clang-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 0 # fetch all history for git diff in clang-tidy - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-clang-test - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 01ed551c..aae2ce98 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: | apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo + DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1 rm cmake-3.28.3-linux-x86_64.tar.gz @@ -69,7 +69,7 @@ jobs: env: CC: gcc-8 CXX: g++-8 - run: ci/scripts/build_paimon.sh $(pwd) + run: ci/scripts/build_paimon.sh --source_dir "$(pwd)" - name: Show ccache statistics if: always() run: ccache -s diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml deleted file mode 100644 index b213f66f..00000000 --- a/.github/workflows/gcc_test.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Gcc Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - gcc-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-gcc-test - - name: Build Paimon - shell: bash - env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml deleted file mode 100644 index 4005c8a2..00000000 --- a/.github/workflows/test_with_sanitizer.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Test with sanitizer - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - test-with-sanitizer: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-sanitizer - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.gitignore b/.gitignore index b71dc569..4d213996 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,7 @@ # Build directories build -build-release -build-debug +build-*/ output release @@ -28,8 +27,20 @@ release .cache # Devcontainer configuration +# Track only *.template files (and subdirectory structure that contains them). .devcontainer/* !.devcontainer/*.template +!.devcontainer/x86_64/ +.devcontainer/x86_64/* +!.devcontainer/x86_64/*.template +# CentOS 7 cross-build image: track raw Dockerfile + helper script (not +# templated because the image is built from the repo root directly). +!.devcontainer/centos7/ +.devcontainer/centos7/* +!.devcontainer/centos7/Dockerfile +!.devcontainer/centos7/run.sh +# rustup-init.bin is a 20 MB prefetched binary — not source, don't commit. +.devcontainer/centos7/rustup-init.bin # Temporary and backup files *~ @@ -53,3 +64,6 @@ FlameGraph # Third party dependencies archives third_party/*.tar.gz + +# Rust / Cargo build artifacts +crates/tantivy_ffi/target/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..9511a133 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,126 @@ + + +# AGENTS.md + +This file provides repository-specific instructions for coding agents working on Paimon C++. +For contributor-facing setup and the complete coding conventions, also read [`CONTRIBUTING.md`](CONTRIBUTING.md) and [`docs/code-style.md`](docs/code-style.md). + +## Scope + +These instructions apply to the entire repository. More deeply nested `AGENTS.md` files, if added later, may provide additional or more specific instructions for their directory trees. + +Keep the requested scope exact. Do not include unrelated refactors, formatting changes, API redesigns, dependency updates, or generated files in a focused change. + +## Repository Layout + +- `include/paimon/`: public C++ API headers. +- `src/paimon/`: core implementation and most unit tests. +- `test/inte/`: end-to-end integration tests. +- `benchmark/`: benchmarks and benchmark-specific tests. +- `examples/`: example programs. +- `docs/` and `apidoc/`: user documentation and API documentation. +- `cmake_modules/` and `build_support/`: CMake helpers and build infrastructure. +- `ci/`: scripts used by continuous integration. +- `test/test_data/`: checked-in test fixtures. +- `third_party/`: third-party sources and patches. +- `build/`, `build-release/`, and `output/`: generated or local build output. + +Do not edit `third_party/`, checked-in fixtures, generated output, or Git LFS objects unless the task explicitly requires it. Never add files from local build directories to a change. + +## Working Rules + +1. Inspect the current implementation, nearby tests, and relevant CMake target before editing. +2. Search for an existing helper or established pattern before adding a new abstraction. +3. Preserve user changes and untracked local files. Do not discard, overwrite, or reformat unrelated work. +4. Prefer the smallest change that fully implements or fixes the requested behavior. +5. Add or update a focused regression test for behavior changes when practical. +6. Do not claim a build or test passed unless the corresponding command completed successfully. +7. Do not commit, push, amend commits, or create a pull request unless explicitly requested. + +When changing a public API, check the declaration under `include/paimon/`, its implementation, symbol visibility, documentation, callers, and tests together. + +## C++ Requirements + +The full rules are in [`docs/code-style.md`](docs/code-style.md). In particular: + +- Use C++17; do not introduce C++20 or later features. +- Use `Status` and `Result` for fallible operations. Do not use exceptions for production error propagation. +- Propagate errors with the project macros, including `PAIMON_RETURN_NOT_OK` and `PAIMON_ASSIGN_OR_RAISE`. +- Use an explicit type, not `auto`, as the declaration in `PAIMON_ASSIGN_OR_RAISE` and `PAIMON_ASSIGN_OR_RAISE_FROM_ARROW`. +- Prefer `std::unique_ptr` for sole ownership and use `std::shared_ptr` only for genuine shared ownership. +- Use `static Create()` plus a private constructor when object initialization can fail. +- Mark new public API symbols with `PAIMON_EXPORT`. +- Reuse helpers under `src/paimon/common/utils/` instead of duplicating utility code. +- Follow `.clang-format`; do not manually restyle unrelated code. +- Add the repository's Apache 2.0 license header to every new source or documentation file. + +## Build and Test + +Use an existing configured build directory when it is compatible with the change. To configure a new debug build with tests: + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DPAIMON_BUILD_TESTS=ON +``` + +Start with the narrowest relevant validation: + +```bash +cmake --build build --target -j "$(nproc)" +./build/debug/ --gtest_filter='' +``` + +Then broaden validation in proportion to the change: + +```bash +# All unit tests +cmake --build build --target unittest -j "$(nproc)" + +# Formatting, lint, and repository checks for changed files +pre-commit run --files +git diff --check +``` + +For changes to build configuration, public APIs, shared infrastructure, or cross-module behavior, run the relevant wider test suite. The CI-equivalent build entry point is `ci/scripts/build_paimon.sh`; it may rebuild all dependencies and take substantially longer than a focused local target. + +If a required check cannot be run because of missing dependencies, unsupported hardware, or time constraints, report exactly what was and was not run. + +## Testing Conventions + +- Use GoogleTest and name test files `*_test.cpp`. +- Place unit tests next to the corresponding implementation unless an existing target establishes another location. +- Extend an existing test target when appropriate instead of creating a new executable for one small test. +- Test externally observable behavior and failure cases; avoid coupling tests to incidental implementation details. +- Use existing test utilities and temporary-directory helpers. Do not write tests that depend on developer-specific absolute paths. +- Keep fixtures deterministic and small. Do not rewrite existing fixture data unless the task explicitly calls for it. +- Use `ASSERT_*` when later assertions depend on the condition succeeding. + +## Delivery + +Before handing off a change: + +1. Review `git diff` for accidental or unrelated edits. +2. Run `git diff --check`. +3. Run the narrowest relevant build and test, plus any wider checks justified by the risk. +4. Summarize the changed behavior and list the exact validation commands that ran. +5. Call out skipped validation, remaining risks, or follow-up work explicitly. +6. When explicitly asked to commit, follow the Conventional Commits requirements in [`CONTRIBUTING.md`](CONTRIBUTING.md#commit-messages-and-pull-request-titles). +7. When explicitly asked to open a pull request, follow [`CONTRIBUTING.md`](CONTRIBUTING.md#commit-messages-and-pull-request-titles), use a Conventional Commits title, and complete every applicable section of [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b6aca77a..52053d4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,11 @@ # Adapted from Apache Arrow: # https://github.com/apache/arrow/blob/apache-arrow-17.0.0/cpp/CMakeLists.txt -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.22) +# 3.22 is the minimum required by Corrosion-rs (used for the Rust<->C++ FFI +# integration, see crates/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and +# CentOS 8+/RHEL 9+ ship 3.20+, so older distributions need a newer CMake +# installation. message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://cmake.org/cmake/help/latest/policy/CMP0135.html @@ -37,7 +41,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() project(paimon - VERSION 0.2.3 + VERSION 0.3.0 DESCRIPTION "Paimon C++ Project") string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE) @@ -50,14 +54,19 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(PAIMON_BUILD_STATIC "Build static library" ON) option(PAIMON_BUILD_SHARED "Build shared library" ON) option(PAIMON_BUILD_TESTS "Build tests" OFF) +option(PAIMON_BUILD_BENCHMARKS "Build benchmarks" OFF) option(PAIMON_USE_ASAN "Use Address Sanitizer" OFF) option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF) option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON) option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON) option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON) option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) +option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF) +option(PAIMON_ENABLE_NETWORK_TESTS + "Whether to enable tests that access real remote services over the network" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) - +option(PAIMON_ENABLE_TANTIVY + "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() @@ -67,6 +76,12 @@ endif() if(PAIMON_ENABLE_JINDO) add_definitions(-DPAIMON_ENABLE_JINDO) endif() +if(PAIMON_ENABLE_S3) + add_definitions(-DPAIMON_ENABLE_S3) +endif() +if(PAIMON_ENABLE_NETWORK_TESTS) + add_definitions(-DPAIMON_ENABLE_NETWORK_TESTS) +endif() if(PAIMON_USE_CXX11_ABI) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1) else() @@ -84,6 +99,10 @@ if(PAIMON_ENABLE_LUCENE) add_definitions(-DPAIMON_ENABLE_LUCENE) endif() +if(PAIMON_ENABLE_TANTIVY) + add_definitions(-DPAIMON_ENABLE_TANTIVY) +endif() + add_definitions(-DSNAPPY_CODEC_AVAILABLE) add_definitions(-DZSTD_CODEC_AVAILABLE) add_definitions(-DRAPIDJSON_HAS_STDSTRING) @@ -286,8 +305,18 @@ set(PAIMON_SHARED_PRIVATE_LINK_LIBS ${PAIMON_STATIC_LINK_LIBS}) add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL) add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL) -list(APPEND PAIMON_LINK_LIBS ${CMAKE_DL_LIBS}) -list(APPEND PAIMON_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS}) +# ---- tantivy-fulltext Rust FFI via Corrosion-rs -------------------------------- +# Corrosion wraps the Cargo crate as a CMake target named `paimon_tantivy_ffi`. +# `corrosion_experimental_cbindgen` runs cbindgen from CMake and writes the +# header to a stable path; it also adds that path to the target's INTERFACE +# include dirs so C++ consumers pick it up via target_link_libraries. +if(PAIMON_ENABLE_TANTIVY) + include(CorrosionFetch) + corrosion_import_crate(MANIFEST_PATH crates/tantivy_ffi/Cargo.toml CRATES + paimon_tantivy_ffi) + corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME + paimon_tantivy_ffi.h) +endif() if(PAIMON_ENABLE_LUCENE) set(PAIMON_DICT_DEST "share/paimon/dict") @@ -325,11 +354,37 @@ add_compile_definitions("GLOG_USE_GLOG_EXPORT") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -set(PAIMON_VERSION_SCRIPT_FLAGS - "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map") +set(PAIMON_VERSION_SCRIPT_FLAGS) +if(NOT APPLE) + set(PAIMON_VERSION_SCRIPT_FLAGS + "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map") +endif() set(ENV{PAIMON_TEST_DATA} "${CMAKE_SOURCE_DIR}/test/test_data") +if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) + resolve_dependency(GTest) + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + + paimon_link_libraries_whole_archive(PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_local_file_system_static) + paimon_link_libraries_no_as_needed(PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS + paimon_local_file_system_shared) + paimon_link_libraries_whole_archive(PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS + paimon_blob_file_format_static) + paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS + paimon_parquet_file_format_static) + + if(PAIMON_ENABLE_ORC) + paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS + paimon_orc_file_format_static) + endif() + if(PAIMON_ENABLE_AVRO) + paimon_link_libraries_whole_archive(PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS + paimon_avro_file_format_static) + endif() +endif() + if(PAIMON_BUILD_TESTS) if(NOT PAIMON_ENABLE_ORC) message(FATAL_ERROR "PAIMON_ENABLE_ORC must be enabled if PAIMON_BUILD_TESTS is enable" @@ -341,7 +396,6 @@ if(PAIMON_BUILD_TESTS) endif() # Adding unit tests part of the "paimon" portion of the test suite add_custom_target(paimon-tests) - resolve_dependency(GTest) add_custom_target(unittest ctest @@ -351,48 +405,80 @@ if(PAIMON_BUILD_TESTS) --output-on-failure) add_dependencies(unittest paimon-tests) - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) include_directories("${CMAKE_SOURCE_DIR}/test/") - set(TEST_STATIC_LINK_LIBS - "-Wl,--whole-archive" + paimon_link_libraries_whole_archive( + TEST_WHOLE_ARCHIVE_LINK_LIBS paimon_file_index_static paimon_global_index_static paimon_local_file_system_static - paimon_mock_file_format_static - "-Wl,--no-whole-archive" - "-Wl,--no-as-needed" - paimon_parquet_file_format_shared - paimon_blob_file_format_shared - "-Wl,--as-needed") + paimon_mock_file_format_static) + paimon_link_libraries_no_as_needed( + TEST_PLUGIN_LINK_LIBS paimon_parquet_file_format_shared + paimon_blob_file_format_shared) + set(TEST_STATIC_LINK_LIBS ${TEST_WHOLE_ARCHIVE_LINK_LIBS} ${TEST_PLUGIN_LINK_LIBS}) + paimon_link_libraries_whole_archive(PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_local_file_system_static) + paimon_link_libraries_no_as_needed(PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS + paimon_local_file_system_shared) + paimon_link_libraries_whole_archive(PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS + paimon_blob_file_format_static) + paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS + paimon_parquet_file_format_static) if(PAIMON_ENABLE_ORC) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_orc_file_format_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_AVRO) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_avro_file_format_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_JINDO) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_jindo_file_system_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_jindo_file_system_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) + endif() + if(PAIMON_ENABLE_S3) + paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_s3_file_system_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_s3_file_system_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_LUMINA) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS + paimon_lumina_index_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_lumina_index_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_LUCENE) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS + paimon_lucene_index_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_lucene_index_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() endif() +if(PAIMON_BUILD_BENCHMARKS) + add_custom_target(paimon-benchmarks) + add_custom_target(benchmark + ctest + -j4 + -L + benchmark + --output-on-failure) + add_dependencies(benchmark paimon-benchmarks) + + set(PAIMON_BENCHMARK_LINK_TOOLCHAIN benchmark::benchmark) +endif() + paimon_print_dependency_resolution_summary() include(CMakePackageConfigHelpers) @@ -411,11 +497,15 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" config_summary_message() +if(PAIMON_ENABLE_S3) + find_package(CURL REQUIRED) +endif() add_subdirectory(src/paimon) add_subdirectory(src/paimon/fs/local) if(PAIMON_ENABLE_JINDO) add_subdirectory(src/paimon/fs/jindo) endif() +add_subdirectory(src/paimon/fs/s3) add_subdirectory(src/paimon/format/blob) add_subdirectory(src/paimon/format/orc) add_subdirectory(src/paimon/format/parquet) @@ -424,6 +514,9 @@ if(PAIMON_ENABLE_LUMINA) add_subdirectory(src/paimon/global_index/lumina) endif() add_subdirectory(src/paimon/global_index/lucene) +if(PAIMON_ENABLE_TANTIVY) + add_subdirectory(src/paimon/global_index/tantivy) +endif() add_subdirectory(src/paimon/testing/mock) add_subdirectory(src/paimon/testing/utils) add_subdirectory(test/inte) @@ -431,3 +524,11 @@ add_subdirectory(test/inte) install(EXPORT PaimonTargets NAMESPACE Paimon:: DESTINATION ${PAIMON_CMAKE_INSTALL_DIR}) + +if(PAIMON_BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() + +install(EXPORT PaimonTargets + NAMESPACE Paimon:: + DESTINATION ${PAIMON_CMAKE_INSTALL_DIR}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 086f4a65..cabeca59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,27 @@ If you find a bug or want to request a feature, please open an [issue](https://g --- +## Commit Messages and Pull Request Titles + +Use the [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages and pull request titles: + +```text +(): +``` + +Examples: + +```text +feat(parquet): support page-level bitmap filtering +fix: handle non-contiguous row ranges +test(executor): add shutdown coverage +docs: update the build instructions +``` + +Choose a type and optional scope that accurately describe the change. Keep the description concise and write it in the imperative mood. A pull request title should summarize the complete change and use the same format. + +--- + ## Submitting Pull Requests 1. **Fork** the repository and create a feature branch from `main`. @@ -37,6 +58,8 @@ If you find a bug or want to request a feature, please open an [issue](https://g 4. Ensure all checks pass. 5. Open a pull request against `main`. Fill in the [PR template](.github/PULL_REQUEST_TEMPLATE.md). +When addressing review feedback or adding follow-up changes to an open pull request, prefer a separate commit instead of amending and force-pushing existing commits. This makes the incremental diff easier for reviewers to inspect. Rewrite existing commits only when a maintainer explicitly requests it. + ### PR Checklist Before submitting, please verify: diff --git a/LICENSE b/LICENSE index 8ef61972..6e39411a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -218,6 +219,7 @@ This product includes code from Apache Iceberg C++. * Dev Container utilities: * .devcontainer/Dockerfile.template * .devcontainer/devcontainer.json.template + * .devcontainer/x86_64/devcontainer.json.template * CI utilities: * .pre-commit-config.yaml * Avro direct decoder/encoder: @@ -233,6 +235,28 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code based on Apache Spark. + +* Variant utilities: + * src/paimon/common/data/variant/generic_variant.cpp + * src/paimon/common/data/variant/generic_variant.h + * src/paimon/common/data/variant/variant_binary_util.cpp + * src/paimon/common/data/variant/variant_binary_util.h + * src/paimon/common/data/variant/variant_builder.cpp + * src/paimon/common/data/variant/variant_builder.h + * src/paimon/common/data/variant/variant_reassembler.cpp + * src/paimon/common/data/variant/variant_reassembler.h + * src/paimon/common/data/variant/variant_schema.h + * src/paimon/common/data/variant/variant_shredding_utils.cpp + * src/paimon/common/data/variant/variant_shredding_writer.cpp + * src/paimon/common/data/variant/variant_shredding_writer.h + +Copyright: 2014 and onwards The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code based on Google Guava. * Preconditions utility in src/paimon/common/utils/preconditions.h @@ -405,6 +429,16 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from LucenePlusPlus. + +* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory + +Copyright: 2009-2014 Alan Wright. +Home page: https://github.com/luceneplusplus/LucenePlusPlus +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code derived from PyTorch TH simd.h. * SIMD detection code in third_party/roaring_bitmap/roaring.cpp diff --git a/NOTICE b/NOTICE index ed0d3839..a12e432f 100644 --- a/NOTICE +++ b/NOTICE @@ -24,6 +24,11 @@ Copyright 2024-2025 The Apache Software Foundation -------------------------------------------------------------------------------- +Apache Spark +Copyright 2014 and onwards The Apache Software Foundation. + +-------------------------------------------------------------------------------- + Apache ORC Copyright 2013 and onwards The Apache Software Foundation. @@ -45,5 +50,10 @@ PalDB -------------------------------------------------------------------------------- +This product includes software from LucenePlusPlus project (Apache 2.0) +Copyright 2009-2014 Alan Wright. + +-------------------------------------------------------------------------------- + JindoSDK NextArch C++ Copyright 2024-present Alibaba Cloud. diff --git a/README.md b/README.md index 57e32d47..d1140f51 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,18 @@ Paimon-cpp currently provides: - **File systems**: file system abstraction with built-in local and Jindo file system support. - **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support. - **Runtime utilities**: memory pool and thread pool abstractions with default implementations. -- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index capabilities including bitmap index, B-tree index, DiskANN-based vector search with Lumina, and Lucene-based full-text search. -- **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, including commit messages, data splits, and manifests. - -The current implementation supports the `x86_64` architecture. +- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index + capabilities including B-tree index, DiskANN-based vector search with Lumina, and Lucene-based + full-text search. +- **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, + including commit messages, data splits, and manifests. + +> **Bitmap global index compatibility:** Java Paimon now uses a dedicated bitmap global index +> format instead of the previously shared wrapped bitmap file index format. +> Paimon C++ therefore currently treats the `bitmap` global index type as unsupported. The legacy +> implementation remains in the codebase pending migration to the Java-compatible format. + +Note: Linux x86_64 and macOS arm64 builds are currently verified. ## Building diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 00000000..375e9896 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(NOT PAIMON_BUILD_BENCHMARKS AND NOT PAIMON_BUILD_TESTS) + return() +endif() + +find_package(Threads REQUIRED) + +set(PAIMON_BENCHMARK_STATIC_LINK_LIBS + paimon_shared ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS} + ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS}) + +if(PAIMON_ENABLE_ORC) + list(APPEND PAIMON_BENCHMARK_STATIC_LINK_LIBS + ${PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS}) +endif() + +if(PAIMON_ENABLE_AVRO) + list(APPEND PAIMON_BENCHMARK_STATIC_LINK_LIBS + ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS}) +endif() + +set(PAIMON_BENCHMARK_PLATFORM_LINK_LIBS) +if(UNIX AND NOT APPLE) + find_library(PAIMON_BENCHMARK_RT_LIBRARY rt) + if(PAIMON_BENCHMARK_RT_LIBRARY) + list(APPEND PAIMON_BENCHMARK_PLATFORM_LINK_LIBS ${PAIMON_BENCHMARK_RT_LIBRARY}) + endif() +endif() + +if(PAIMON_BUILD_BENCHMARKS) + add_paimon_benchmark(read_write_benchmark + SOURCES + benchmark_helpers.cpp + benchmark_suite.cpp + benchmark_case_write.cpp + benchmark_case_read.cpp + benchmark_case_pk_write.cpp + benchmark_case_mor_read.cpp + read_write_benchmark.cpp + STATIC_LINK_LIBS + arrow + parquet + ${PAIMON_BENCHMARK_STATIC_LINK_LIBS} + test_utils_static + Threads::Threads + ${PAIMON_BENCHMARK_PLATFORM_LINK_LIBS} + ${PAIMON_BENCHMARK_LINK_TOOLCHAIN} + EXTRA_INCLUDES + ${CMAKE_SOURCE_DIR}) +endif() + +if(PAIMON_BUILD_TESTS) + add_paimon_test(cli_option_parsing_test + SOURCES + cli_option_parsing_test.cpp + EXTRA_INCLUDES + ${CMAKE_SOURCE_DIR} + STATIC_LINK_LIBS + paimon_shared + ${GTEST_LINK_TOOLCHAIN}) +endif() diff --git a/benchmark/benchmark_case_mor_read.cpp b/benchmark/benchmark_case_mor_read.cpp new file mode 100644 index 00000000..b5d8ea62 --- /dev/null +++ b/benchmark/benchmark_case_mor_read.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_MOR_Read(::benchmark::State& state) { + paimon::benchmark::RunBMMorRead(state); +} + +} // namespace + +BENCHMARK(BM_MOR_Read) + ->ArgNames({"prefetch_parallel"}) + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Args({1}) + ->Args({2}) + ->Args({4}); diff --git a/benchmark/benchmark_case_pk_write.cpp b/benchmark/benchmark_case_pk_write.cpp new file mode 100644 index 00000000..546e7c7a --- /dev/null +++ b/benchmark/benchmark_case_pk_write.cpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_PK_Write(::benchmark::State& state) { + paimon::benchmark::RunBMPkWrite(state); +} + +} // namespace + +BENCHMARK(BM_PK_Write)->Unit(benchmark::kMillisecond)->UseRealTime(); diff --git a/benchmark/benchmark_case_read.cpp b/benchmark/benchmark_case_read.cpp new file mode 100644 index 00000000..583361d9 --- /dev/null +++ b/benchmark/benchmark_case_read.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_Read(::benchmark::State& state) { + paimon::benchmark::RunBMRead(state); +} + +} // namespace + +BENCHMARK(BM_Read) + ->ArgNames({"prefetch_parallel"}) + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Args({1}) + ->Args({2}) + ->Args({4}); diff --git a/benchmark/benchmark_case_write.cpp b/benchmark/benchmark_case_write.cpp new file mode 100644 index 00000000..5a60e660 --- /dev/null +++ b/benchmark/benchmark_case_write.cpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_Write(::benchmark::State& state) { + paimon::benchmark::RunBMWrite(state); +} + +} // namespace + +BENCHMARK(BM_Write)->Unit(benchmark::kMillisecond)->UseRealTime(); diff --git a/benchmark/benchmark_helpers.cpp b/benchmark/benchmark_helpers.cpp new file mode 100644 index 00000000..0b71d072 --- /dev/null +++ b/benchmark/benchmark_helpers.cpp @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_helpers.h" + +#include + +#include "benchmark/benchmark.h" +#include "fmt/format.h" + +namespace paimon::benchmark { + +bool BenchmarkHelpers::ValidateFileFormatOrSkip(::benchmark::State& state, + const std::string& file_format, bool is_supported, + SkipFn skip) { + if (!is_supported) { + skip(state, fmt::format("file format is not supported in this build: {}", file_format)); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidateSourcePresenceOrSkip(::benchmark::State& state, + const std::string& source_path, + const std::string& message, SkipFn skip) { + if (source_path.empty()) { + skip(state, message); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidateSourceSupportOrSkip(::benchmark::State& state, + const std::string& source_format, + bool is_supported, SkipFn skip) { + if (!is_supported) { + skip(state, + fmt::format("source data mode requires reader support in this build for format: {}", + source_format)); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidatePrefetchParallelOrSkip(::benchmark::State& state, + int32_t prefetch_parallel_num, SkipFn skip) { + if (prefetch_parallel_num <= 0) { + skip(state, "prefetch_parallel must be greater than 0"); + return false; + } + return true; +} + +Result BenchmarkHelpers::RunReadIterations(::benchmark::State& state, + const ReadOnceFn& read_once) { + int64_t rows_read = 0; + for (auto _ : state) { + PAIMON_ASSIGN_OR_RAISE(rows_read, read_once()); + } + return rows_read; +} + +Result BenchmarkHelpers::TryRunSourceTableReadMode(::benchmark::State& state, + const std::string& benchmark_name, + const std::string& source_table_path, + const ReadOnceFn& read_once) { + if (source_table_path.empty()) { + return false; + } + + std::cout << "[benchmark][" << benchmark_name << "] source_table_path=" << source_table_path + << std::endl; + PAIMON_ASSIGN_OR_RAISE(const int64_t rows_read, RunReadIterations(state, read_once)); + state.SetItemsProcessed(state.iterations() * rows_read); + return true; +} + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_helpers.h b/benchmark/benchmark_helpers.h new file mode 100644 index 00000000..f54f199d --- /dev/null +++ b/benchmark/benchmark_helpers.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace benchmark { +class State; +} + +namespace paimon::benchmark { + +class BenchmarkHelpers { + public: + using ReadOnceFn = std::function()>; + using SkipFn = void (*)(::benchmark::State&, const std::string&); + + static bool ValidateFileFormatOrSkip(::benchmark::State& state, const std::string& file_format, + bool is_supported, SkipFn skip); + + static bool ValidateSourcePresenceOrSkip(::benchmark::State& state, + const std::string& source_path, + const std::string& message, SkipFn skip); + + static bool ValidateSourceSupportOrSkip(::benchmark::State& state, + const std::string& source_format, bool is_supported, + SkipFn skip); + + static bool ValidatePrefetchParallelOrSkip(::benchmark::State& state, + int32_t prefetch_parallel_num, SkipFn skip); + + static Result RunReadIterations(::benchmark::State& state, + const ReadOnceFn& read_once); + + static Result TryRunSourceTableReadMode(::benchmark::State& state, + const std::string& benchmark_name, + const std::string& source_table_path, + const ReadOnceFn& read_once); +}; + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp new file mode 100644 index 00000000..0a48a917 --- /dev/null +++ b/benchmark/benchmark_suite.cpp @@ -0,0 +1,882 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/io/api.h" +#include "benchmark/benchmark_helpers.h" +#include "benchmark/cli_option_parsing.h" +#include "paimon/api.h" +#include "paimon/catalog/catalog.h" +#include "paimon/testing/utils/testharness.h" + +#if __has_include("parquet/arrow/reader.h") +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#define PAIMON_BENCHMARK_HAS_PARQUET_READER 1 +#else +#define PAIMON_BENCHMARK_HAS_PARQUET_READER 0 +#endif + +namespace paimon::benchmark { + +namespace { + +constexpr int64_t kSourceBatchMaxRows = 4096; +constexpr int32_t kRowToBatchThreadNumber = 3; + +struct BenchmarkCliOptions { + std::string source_data_file; + std::string source_table_path; + std::vector pk_columns; + std::vector> extra_options; +}; + +struct SourceDataSpec { + std::string format; + std::string path; +}; + +BenchmarkCliOptions& MutableBenchmarkCliOptions() { + static BenchmarkCliOptions options; + return options; +} + +const BenchmarkCliOptions& GetBenchmarkCliOptions() { + return MutableBenchmarkCliOptions(); +} + +Status ParsePaimonBenchmarkCliArgsImpl(int32_t* argc, char** argv) { + auto& options = MutableBenchmarkCliOptions(); + options = BenchmarkCliOptions{}; + const int32_t parsed_argc = *argc; + int32_t write_index = 1; + for (int32_t arg_index = 1; arg_index < parsed_argc; ++arg_index) { + const std::string arg(argv[arg_index]); + + PAIMON_ASSIGN_OR_RAISE(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + parsed_argc, argv, arg, "--paimon_source_data_file", &arg_index, + &options.source_data_file)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseStringOptionArg( + parsed_argc, argv, arg, "--paimon_source_table_path", + &arg_index, &options.source_table_path)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseCommaSeparatedOptionArg( + parsed_argc, argv, arg, "--paimon_pk_columns", + &arg_index, &options.pk_columns)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseDelimitedRepeatableOptionArg( + parsed_argc, argv, arg, "--paimon_option", &arg_index, + &options.extra_options)); + if (is_parsed) { + continue; + } + + argv[write_index++] = argv[arg_index]; + } + + *argc = write_index; + argv[write_index] = nullptr; + return Status::OK(); +} + +bool HasHelpFlagImpl(int32_t argc, char** argv) { + for (int32_t arg_index = 1; arg_index < argc; ++arg_index) { + const std::string arg(argv[arg_index]); + if (arg == "-h" || arg == "--help" || arg == "--help=true") { + return true; + } + } + return false; +} + +void PrintPaimonBenchmarkCliHelpImpl() { + std::cout + << "Paimon benchmark custom options:\n" + << " --paimon_source_data_file=\n" + << " Required. External source data file used to build benchmark data.\n" + << " Currently supports Parquet source files.\n" + << " Also supports: --paimon_source_data_file \n" + << " --paimon_source_table_path=\n" + << " Optional for BM_Read and BM_MOR_Read. If set, read directly from existing\n" + << " table path and skip source file loading and pre-write stage.\n" + << " Also supports: --paimon_source_table_path \n" + << " --paimon_pk_columns=\n" + << " Required by BM_PK_Write and BM_MOR_Read.\n" + << " Also supports: --paimon_pk_columns \n" + << " --paimon_option=:;:\n" + << " Optional and repeatable. Pass through table options as-is.\n" + << " Default table file format is parquet; use file.format: to override.\n" + << " Also supports: --paimon_option :;:\n" + << " Note: use quotes in shell, e.g. \"--paimon_option k1:v1;k2:v2\".\n" + << "\n" + << "Example:\n" + << " paimon-read-write-benchmark --paimon_source_data_file /path/data.parquet \\\n" + << " --paimon_pk_columns=id --paimon_option \"read.batch-size:8192\" \\\n" + << " --benchmark_filter=BM_Read\n" + << std::endl; +} + +Result> CreateBenchmarkWorkspace() { + auto workspace = paimon::test::UniqueTestDirectory::Create(); + if (workspace == nullptr) { + return Status::Invalid("failed to create benchmark workspace"); + } + return workspace; +} + +uint64_t NextTableId() { + static std::atomic id{0}; + return ++id; +} + +std::string RequirePath(const std::string& root_path, const std::string& db_name, + const std::string& table_name) { + return root_path + "/" + db_name + ".db/" + table_name; +} + +template +Result AddContext(paimon::Result&& result, const std::string& context) { + if (!result.ok()) { + const Status status = result.status(); + return status.WithMessage(context, ": ", status.message()); + } + return std::move(result).value(); +} + +Status AddContext(const paimon::Status& status, const std::string& context) { + if (!status.ok()) { + return status.WithMessage(context, ": ", status.message()); + } + return Status::OK(); +} + +void SkipWithMessage(::benchmark::State& state, const std::string& message) { + state.SkipWithError(message); +} + +std::string GetConfiguredFileFormat() { + std::string file_format = "parquet"; + for (const auto& kv : GetBenchmarkCliOptions().extra_options) { + if (kv.first == paimon::Options::FILE_FORMAT) { + file_format = kv.second; + } + } + return file_format; +} + +bool IsFileFormatSupported(const std::string& format) { + if (format == "parquet") { + return true; + } + if (format == "orc") { +#ifdef PAIMON_ENABLE_ORC + return true; +#else + return false; +#endif + } + return false; +} + +void ApplyExtraOptions(std::map* options) { + for (const auto& kv : GetBenchmarkCliOptions().extra_options) { + (*options)[kv.first] = kv.second; + } +} + +std::map BuildOptions(const std::string& file_format) { + std::map options = { + {paimon::Options::FILE_FORMAT, file_format}, + }; + ApplyExtraOptions(&options); + return options; +} + +std::map BuildPkOptions(const std::string& file_format) { + auto options = BuildOptions(file_format); + options[paimon::Options::BUCKET] = "1"; + options[paimon::Options::MERGE_ENGINE] = "deduplicate"; + return options; +} + +std::string GetSourceDataFilePath() { + return GetBenchmarkCliOptions().source_data_file; +} + +std::string GetSourceTablePath() { + return GetBenchmarkCliOptions().source_table_path; +} + +const std::vector& GetPkColumns() { + return GetBenchmarkCliOptions().pk_columns; +} + +SourceDataSpec GetSourceDataSpec() { + const std::string source_data_file_path = GetSourceDataFilePath(); + if (!source_data_file_path.empty()) { + return {"parquet", source_data_file_path}; + } + return {"", ""}; +} + +int64_t GetSourceBatchMaxRows() { + return kSourceBatchMaxRows; +} + +int32_t GetRowToBatchThreadNumber() { + return kRowToBatchThreadNumber; +} + +bool SupportsParquetSourceDataMode() { +#if PAIMON_BENCHMARK_HAS_PARQUET_READER + return true; +#else + return false; +#endif +} + +bool SupportsSourceDataMode(const std::string& source_format) { + if (source_format == "parquet") { + return SupportsParquetSourceDataMode(); + } + return false; +} + +struct SourceDataMetadata { + std::shared_ptr schema; + int64_t total_rows = 0; + std::string format; + std::string path; +}; + +#if PAIMON_BENCHMARK_HAS_PARQUET_READER +Result> OpenParquetSourceReader( + const std::string& path) { + auto input = arrow::io::ReadableFile::Open(path); + if (!input.ok()) { + return Status::Invalid("open Parquet source failed: ", path, ", ", + input.status().ToString()); + } + + std::unique_ptr parquet_reader; + const auto open_status = parquet::arrow::OpenFile( + input.ValueUnsafe(), arrow::default_memory_pool(), &parquet_reader); + if (!open_status.ok()) { + return Status::Invalid("create Parquet reader failed: ", open_status.ToString()); + } + parquet_reader->set_batch_size(GetSourceBatchMaxRows()); + return parquet_reader; +} +#endif + +Result LoadParquetSourceMetadata(const std::string& path) { +#if !PAIMON_BENCHMARK_HAS_PARQUET_READER + return Status::Invalid( + "Parquet source data mode requires parquet::arrow reader support in this build"); +#else + static SourceDataMetadata cache; + static std::mutex cache_mutex; + std::lock_guard lock(cache_mutex); + if (cache.path == path && cache.format == "parquet") { + return cache; + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr parquet_reader, + OpenParquetSourceReader(path)); + std::shared_ptr schema; + const auto schema_status = parquet_reader->GetSchema(&schema); + if (!schema_status.ok()) { + return Status::Invalid("read Parquet source schema failed: ", schema_status.ToString()); + } + + const int64_t total_rows = parquet_reader->parquet_reader()->metadata()->num_rows(); + if (total_rows <= 0) { + return Status::Invalid("Parquet source is empty: ", path); + } + + cache.schema = std::move(schema); + cache.total_rows = total_rows; + cache.format = "parquet"; + cache.path = path; + return cache; +#endif +} + +Result LoadSourceDataMetadata(const SourceDataSpec& source_spec) { + if (source_spec.format == "parquet") { + return LoadParquetSourceMetadata(source_spec.path); + } + return Status::Invalid("unknown source format: ", source_spec.format); +} + +std::shared_ptr BuildStructArrayFromRecordBatch( + const std::shared_ptr& batch) { + return std::make_shared(arrow::struct_(batch->schema()->fields()), + batch->num_rows(), batch->columns()); +} + +Result> MakeRecordBatch( + const std::shared_ptr& arr) { + ArrowArray c_array; + if (!arrow::ExportArray(*arr, &c_array).ok()) { + return Status::Invalid("failed to export arrow array"); + } + paimon::RecordBatchBuilder builder(&c_array); + return AddContext(builder.Finish(), "build paimon record batch"); +} + +Status EnsureTable(const std::string& root_path, const std::string& db_name, + const std::string& table_name, const std::map& options, + const std::shared_ptr& schema, + const std::vector& primary_keys = {}) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr catalog, + AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); + PAIMON_RETURN_NOT_OK( + AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); + + ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export table schema"); + } + PAIMON_RETURN_NOT_OK( + AddContext(catalog->CreateTable(paimon::Identifier(db_name, table_name), &c_schema, + /*partition_keys=*/{}, primary_keys, options, + /*ignore_if_exists=*/false), + "create table")); + return Status::OK(); +} + +Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataSpec& source_spec) { + if (source_spec.format != "parquet") { + return Status::Invalid("unknown source format: ", source_spec.format); + } + +#if !PAIMON_BENCHMARK_HAS_PARQUET_READER + return Status::Invalid( + "Parquet source data mode requires parquet::arrow reader support in this build"); +#else + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr parquet_reader, + OpenParquetSourceReader(source_spec.path)); + std::unique_ptr batch_reader; + const auto reader_status = parquet_reader->GetRecordBatchReader(&batch_reader); + if (!reader_status.ok()) { + return Status::Invalid("create Parquet source batch reader failed: ", + reader_status.ToString()); + } + + int64_t written_rows = 0; + while (true) { + std::shared_ptr record_batch; + const auto read_status = batch_reader->ReadNext(&record_batch); + if (!read_status.ok()) { + return Status::Invalid("read Parquet source batch failed: ", read_status.ToString()); + } + if (record_batch == nullptr) { + break; + } + if (record_batch->num_rows() <= 0) { + continue; + } + + auto struct_array = BuildStructArrayFromRecordBatch(record_batch); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + MakeRecordBatch(struct_array)); + PAIMON_RETURN_NOT_OK(AddContext(writer->Write(std::move(batch)), "write batch")); + written_rows += record_batch->num_rows(); + } + + if (written_rows <= 0) { + return Status::Invalid("source file has no non-empty data batches: ", source_spec.path); + } + return Status::OK(); +#endif +} + +Status WriteAndCommit(const std::string& table_path, + const std::map& options, + const SourceDataSpec& source_spec) { + paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_ctx, + AddContext(write_builder.SetOptions(options).Finish(), "create write context")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), + "create file store writer")); + + PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); + PAIMON_ASSIGN_OR_RAISE(std::vector> messages, + AddContext(writer->PrepareCommit(), "prepare commit")); + + paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit_ctx, + AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); + return Status::OK(); +} + +struct SharedReadTableCache { + std::string key; + std::unique_ptr workspace; + std::string table_path; + int64_t total_rows = 0; +}; + +struct SharedMorReadTableCache { + std::string key; + std::unique_ptr workspace; + std::string table_path; + int64_t total_rows = 0; +}; + +std::string BuildReadTableCacheKey(const std::string& file_format, + const SourceDataSpec& source_spec) { + return file_format + "|" + source_spec.format + "|" + source_spec.path + "|" + + std::to_string(GetSourceBatchMaxRows()); +} + +std::string JoinColumns(const std::vector& columns) { + std::string joined; + for (size_t i = 0; i < columns.size(); ++i) { + if (i > 0) { + joined.append(","); + } + joined.append(columns[i]); + } + return joined; +} + +Result GetOrCreateSharedMorReadTable( + const std::string& file_format, const SourceDataSpec& source_spec) { + static SharedMorReadTableCache cache; + static std::mutex cache_mutex; + + const std::vector& pk_columns = GetPkColumns(); + const std::string cache_key = + BuildReadTableCacheKey(file_format, source_spec) + "|pk=" + JoinColumns(pk_columns); + std::lock_guard lock(cache_mutex); + if (cache.workspace != nullptr && cache.key == cache_key) { + std::cout << "[benchmark][mor-read] reuse_output_table_path=" << cache.table_path + << std::endl; + return &cache; + } + + auto options = BuildPkOptions(file_format); + PAIMON_ASSIGN_OR_RAISE(const SourceDataMetadata source_metadata, + LoadSourceDataMetadata(source_spec)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr workspace, + CreateBenchmarkWorkspace()); + const std::string db_name = "bench_db"; + const std::string table_name = "mor_read_shared_" + std::to_string(NextTableId()); + PAIMON_RETURN_NOT_OK(EnsureTable(workspace->Str(), db_name, table_name, options, + source_metadata.schema, + /*primary_keys=*/pk_columns)); + const std::string table_path = RequirePath(workspace->Str(), db_name, table_name); + std::cout << "[benchmark][mor-read] create_shared_output_table_path=" << table_path + << std::endl; + PAIMON_RETURN_NOT_OK(WriteAndCommit(table_path, options, source_spec)); + + cache.key = cache_key; + cache.workspace = std::move(workspace); + cache.table_path = table_path; + cache.total_rows = source_metadata.total_rows; + return &cache; +} + +Result GetOrCreateSharedReadTable(const std::string& file_format, + const SourceDataSpec& source_spec) { + static SharedReadTableCache cache; + static std::mutex cache_mutex; + + const std::string cache_key = BuildReadTableCacheKey(file_format, source_spec); + std::lock_guard lock(cache_mutex); + if (cache.workspace != nullptr && cache.key == cache_key) { + std::cout << "[benchmark][read] reuse_output_table_path=" << cache.table_path << std::endl; + return &cache; + } + + auto options = BuildOptions(file_format); + PAIMON_ASSIGN_OR_RAISE(const SourceDataMetadata source_metadata, + LoadSourceDataMetadata(source_spec)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr workspace, + CreateBenchmarkWorkspace()); + const std::string db_name = "bench_db"; + const std::string table_name = "read_shared_" + std::to_string(NextTableId()); + PAIMON_RETURN_NOT_OK( + EnsureTable(workspace->Str(), db_name, table_name, options, source_metadata.schema)); + const std::string table_path = RequirePath(workspace->Str(), db_name, table_name); + std::cout << "[benchmark][read] create_shared_output_table_path=" << table_path << std::endl; + PAIMON_RETURN_NOT_OK(WriteAndCommit(table_path, options, source_spec)); + + cache.key = cache_key; + cache.workspace = std::move(workspace); + cache.table_path = table_path; + cache.total_rows = source_metadata.total_rows; + return &cache; +} + +Result ReadRows(const std::string& table_path, + const std::map& options, + int32_t prefetch_parallel_num) { + paimon::ScanContextBuilder scan_builder(table_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_ctx, + AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scanner, + AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + AddContext(scanner->CreatePlan(), "create plan")); + + paimon::ReadContextBuilder read_builder(table_path); + constexpr int32_t kPrefetchBatchCount = 600; + read_builder.SetOptions(options) + .EnablePrefetch(true) + .SetPrefetchBatchCount(kPrefetchBatchCount) + .SetPrefetchMaxParallelNum(prefetch_parallel_num) + .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) + .SetRowToBatchThreadNumber(GetRowToBatchThreadNumber()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_ctx, + AddContext(read_builder.Finish(), "create read context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); + + int64_t total_rows = 0; + while (true) { + PAIMON_ASSIGN_OR_RAISE(paimon::BatchReader::ReadBatch batch, + AddContext(batch_reader->NextBatch(), "read next batch")); + if (paimon::BatchReader::IsEofBatch(batch)) { + break; + } + auto& [array, schema] = batch; + auto imported = arrow::ImportArray(array.get(), schema.get()); + if (!imported.ok()) { + return Status::Invalid("import c data array failed: ", imported.status().ToString()); + } + total_rows += imported.ValueUnsafe()->length(); + } + + return total_rows; +} + +struct PreparedSourceData { + std::shared_ptr schema; + int64_t total_rows = 0; +}; + +bool TryGetSourceSpec(::benchmark::State& state, SourceDataSpec* source_spec) { + (void)state; + *source_spec = GetSourceDataSpec(); + return true; +} + +bool TryPrepareSourceData(::benchmark::State& state, const SourceDataSpec& source_spec, + PreparedSourceData* prepared) { + auto source_metadata = LoadSourceDataMetadata(source_spec); + if (!source_metadata.ok()) { + SkipWithMessage(state, source_metadata.status().ToString()); + return false; + } + prepared->schema = source_metadata.value().schema; + prepared->total_rows = source_metadata.value().total_rows; + return true; +} + +} // namespace + +Status ParsePaimonBenchmarkCliArgs(int* argc, char** argv) { + auto parsed_argc = static_cast(*argc); + PAIMON_RETURN_NOT_OK(ParsePaimonBenchmarkCliArgsImpl(&parsed_argc, argv)); + *argc = static_cast(parsed_argc); + return Status::OK(); +} + +bool HasHelpFlag(int32_t argc, char** argv) { + return HasHelpFlagImpl(argc, argv); +} + +void PrintPaimonBenchmarkCliHelp() { + PrintPaimonBenchmarkCliHelpImpl(); +} + +void RunBMWrite(::benchmark::State& state) { + const std::string file_format = GetConfiguredFileFormat(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, "--paimon_source_data_file is required", &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + + auto options = BuildOptions(file_format); + PreparedSourceData prepared; + if (!TryPrepareSourceData(state, source_spec, &prepared)) { + return; + } + auto workspace = CreateBenchmarkWorkspace(); + if (!workspace.ok()) { + SkipWithMessage(state, workspace.status().ToString()); + return; + } + + for (auto _ : state) { + const std::string db_name = "bench_db"; + const std::string table_name = "write_" + std::to_string(NextTableId()); + const Status ensure_status = + EnsureTable(workspace.value()->Str(), db_name, table_name, options, prepared.schema); + if (!ensure_status.ok()) { + SkipWithMessage(state, ensure_status.ToString()); + return; + } + const std::string table_path = RequirePath(workspace.value()->Str(), db_name, table_name); + std::cout << "[benchmark][write] output_table_path=" << table_path << std::endl; + const Status write_status = WriteAndCommit(table_path, options, source_spec); + if (!write_status.ok()) { + SkipWithMessage(state, write_status.ToString()); + return; + } + } + + state.SetItemsProcessed(state.iterations() * prepared.total_rows); +} + +void RunBMRead(::benchmark::State& state) { + const auto prefetch_parallel_num = static_cast(state.range(0)); + const std::string file_format = GetConfiguredFileFormat(); + const std::string source_table_path = GetSourceTablePath(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + + if (!BenchmarkHelpers::ValidatePrefetchParallelOrSkip(state, prefetch_parallel_num, + &SkipWithMessage)) { + return; + } + + auto options = BuildOptions(file_format); + + auto source_table_read_result = BenchmarkHelpers::TryRunSourceTableReadMode( + state, "read", source_table_path, + [&]() { return ReadRows(source_table_path, options, prefetch_parallel_num); }); + if (!source_table_read_result.ok()) { + SkipWithMessage(state, source_table_read_result.status().ToString()); + return; + } + if (source_table_read_result.value()) { + return; + } + + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, + "--paimon_source_data_file is required when --paimon_source_table_path is not set", + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + + auto shared_table = GetOrCreateSharedReadTable(file_format, source_spec); + if (!shared_table.ok()) { + SkipWithMessage(state, shared_table.status().ToString()); + return; + } + + auto rows_read = BenchmarkHelpers::RunReadIterations(state, [&]() { + return ReadRows(shared_table.value()->table_path, options, prefetch_parallel_num); + }); + if (!rows_read.ok()) { + SkipWithMessage(state, rows_read.status().ToString()); + return; + } + + state.SetItemsProcessed(state.iterations() * rows_read.value()); +} + +void RunBMPkWrite(::benchmark::State& state) { + const std::string file_format = GetConfiguredFileFormat(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, "--paimon_source_data_file is required", &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + const std::vector& pk_columns = GetPkColumns(); + if (pk_columns.empty()) { + SkipWithMessage(state, "--paimon_pk_columns is required for BM_PK_Write"); + return; + } + + auto options = BuildPkOptions(file_format); + PreparedSourceData prepared; + if (!TryPrepareSourceData(state, source_spec, &prepared)) { + return; + } + auto workspace = CreateBenchmarkWorkspace(); + if (!workspace.ok()) { + SkipWithMessage(state, workspace.status().ToString()); + return; + } + + for (auto _ : state) { + const std::string db_name = "bench_db"; + const std::string table_name = "pk_write_" + std::to_string(NextTableId()); + const Status ensure_status = + EnsureTable(workspace.value()->Str(), db_name, table_name, options, prepared.schema, + /*primary_keys=*/pk_columns); + if (!ensure_status.ok()) { + SkipWithMessage(state, ensure_status.ToString()); + return; + } + const std::string table_path = RequirePath(workspace.value()->Str(), db_name, table_name); + std::cout << "[benchmark][pk-write] output_table_path=" << table_path << std::endl; + const Status write_status = WriteAndCommit(table_path, options, source_spec); + if (!write_status.ok()) { + SkipWithMessage(state, write_status.ToString()); + return; + } + } + + state.SetItemsProcessed(state.iterations() * prepared.total_rows); +} + +void RunBMMorRead(::benchmark::State& state) { + const auto prefetch_parallel_num = static_cast(state.range(0)); + const std::string file_format = GetConfiguredFileFormat(); + const std::string source_table_path = GetSourceTablePath(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidatePrefetchParallelOrSkip(state, prefetch_parallel_num, + &SkipWithMessage)) { + return; + } + + const auto source_table_read_options = BuildOptions(file_format); + auto source_table_read_result = + BenchmarkHelpers::TryRunSourceTableReadMode(state, "mor-read", source_table_path, [&]() { + return ReadRows(source_table_path, source_table_read_options, prefetch_parallel_num); + }); + if (!source_table_read_result.ok()) { + SkipWithMessage(state, source_table_read_result.status().ToString()); + return; + } + if (source_table_read_result.value()) { + return; + } + + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, + "--paimon_source_data_file is required when --paimon_source_table_path is not set", + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (GetPkColumns().empty()) { + SkipWithMessage(state, "--paimon_pk_columns is required for BM_MOR_Read"); + return; + } + + auto options = BuildPkOptions(file_format); + auto shared_table = GetOrCreateSharedMorReadTable(file_format, source_spec); + if (!shared_table.ok()) { + SkipWithMessage(state, shared_table.status().ToString()); + return; + } + + auto rows_read = BenchmarkHelpers::RunReadIterations(state, [&]() { + return ReadRows(shared_table.value()->table_path, options, prefetch_parallel_num); + }); + if (!rows_read.ok()) { + SkipWithMessage(state, rows_read.status().ToString()); + return; + } + state.SetItemsProcessed(state.iterations() * rows_read.value()); +} + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_suite.h b/benchmark/benchmark_suite.h new file mode 100644 index 00000000..69814cea --- /dev/null +++ b/benchmark/benchmark_suite.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "benchmark/benchmark.h" +#include "paimon/status.h" + +namespace paimon::benchmark { + +Status ParsePaimonBenchmarkCliArgs(int* argc, char** argv); +bool HasHelpFlag(int32_t argc, char** argv); +void PrintPaimonBenchmarkCliHelp(); + +void RunBMWrite(::benchmark::State& state); +void RunBMRead(::benchmark::State& state); +void RunBMPkWrite(::benchmark::State& state); +void RunBMMorRead(::benchmark::State& state); + +} // namespace paimon::benchmark diff --git a/benchmark/cli_option_parsing.h b/benchmark/cli_option_parsing.h new file mode 100644 index 00000000..70aea5c6 --- /dev/null +++ b/benchmark/cli_option_parsing.h @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon::benchmark { + +using ParsedOptions = std::vector>; + +inline bool ConsumeCliOption(const std::string& arg, const std::string& option_name, + std::string* value_out) { + const std::string prefix = option_name + "="; + if (arg.rfind(prefix, 0) != 0) { + return false; + } + *value_out = arg.substr(prefix.size()); + return true; +} + +inline std::string TrimAsciiWhitespace(const std::string& value) { + const auto first = value.find_first_not_of(" \t\n\r"); + if (first == std::string::npos) { + return ""; + } + const auto last = value.find_last_not_of(" \t\n\r"); + return value.substr(first, last - first + 1); +} + +inline Result> ParseCommaSeparatedColumns(const std::string& input, + const std::string& option_name) { + if (input.empty()) { + return Status::Invalid("missing value for ", option_name); + } + + std::vector columns; + size_t segment_start = 0; + for (size_t index = 0; index <= input.size(); ++index) { + if (index != input.size() && input[index] != ',') { + continue; + } + + const std::string column = + TrimAsciiWhitespace(input.substr(segment_start, index - segment_start)); + if (column.empty()) { + return Status::Invalid("invalid ", option_name, ": empty column name"); + } + columns.push_back(column); + segment_start = index + 1; + } + return columns; +} + +inline Result ParseDelimitedOptions(const std::string& input, + const std::string& option_name) { + if (input.empty()) { + return Status::Invalid("missing value for ", option_name); + } + + ParsedOptions parsed; + std::string token; + for (size_t index = 0; index <= input.size(); ++index) { + const bool at_end = (index == input.size()); + if (!at_end && input[index] != ';') { + token.push_back(input[index]); + continue; + } + + const std::string segment = TrimAsciiWhitespace(token); + if (segment.empty()) { + return Status::Invalid("invalid ", option_name, ": empty option segment"); + } + + const auto separator = segment.find(':'); + if (separator == std::string::npos) { + return Status::Invalid("invalid ", option_name, ": expected key:value"); + } + + const std::string key = TrimAsciiWhitespace(segment.substr(0, separator)); + const std::string value = TrimAsciiWhitespace(segment.substr(separator + 1)); + if (key.empty() || value.empty()) { + return Status::Invalid("invalid ", option_name, ": expected key:value"); + } + + parsed.emplace_back(key, value); + token.clear(); + } + return parsed; +} + +inline Result ParseStringOptionArg(int32_t argc, char** argv, const std::string& arg, + const std::string& option_name, int32_t* arg_index, + std::string* value_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + *value_out = std::move(parsed_value); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + *value_out = argv[++(*arg_index)]; + return true; +} + +inline Result ParseCommaSeparatedOptionArg(int32_t argc, char** argv, const std::string& arg, + const std::string& option_name, int32_t* arg_index, + std::vector* columns_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + PAIMON_ASSIGN_OR_RAISE(*columns_out, ParseCommaSeparatedColumns(parsed_value, option_name)); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + PAIMON_ASSIGN_OR_RAISE( + *columns_out, ParseCommaSeparatedColumns(std::string(argv[++(*arg_index)]), option_name)); + return true; +} + +inline Result ParseDelimitedRepeatableOptionArg( + int32_t argc, char** argv, const std::string& arg, const std::string& option_name, + int32_t* arg_index, std::vector>* options_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + ParsedOptions parsed_options; + PAIMON_ASSIGN_OR_RAISE(parsed_options, ParseDelimitedOptions(parsed_value, option_name)); + options_out->insert(options_out->end(), parsed_options.begin(), parsed_options.end()); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + + const std::string option_arg = argv[++(*arg_index)]; + ParsedOptions parsed_options; + PAIMON_ASSIGN_OR_RAISE(parsed_options, ParseDelimitedOptions(option_arg, option_name)); + options_out->insert(options_out->end(), parsed_options.begin(), parsed_options.end()); + return true; +} + +} // namespace paimon::benchmark diff --git a/benchmark/cli_option_parsing_test.cpp b/benchmark/cli_option_parsing_test.cpp new file mode 100644 index 00000000..5d0c56ed --- /dev/null +++ b/benchmark/cli_option_parsing_test.cpp @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/cli_option_parsing.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::testing { +namespace { + +struct ArgvHolder { + std::vector args; + std::vector argv; + + explicit ArgvHolder(std::vector in_args) : args(std::move(in_args)) { + argv.reserve(args.size()); + for (auto& arg : args) { + argv.push_back(arg.data()); + } + } + + int32_t argc() const { + return static_cast(argv.size()); + } +}; + +TEST(CliOptionParsingTest, ConsumeCliOptionWorks) { + std::string value; + ASSERT_TRUE(paimon::benchmark::ConsumeCliOption("--foo=bar", "--foo", &value)); + ASSERT_EQ(value, "bar"); + + value.clear(); + ASSERT_FALSE(paimon::benchmark::ConsumeCliOption("--foo", "--foo", &value)); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedColumnsWorks) { + ASSERT_OK_AND_ASSIGN(auto parsed, + paimon::benchmark::ParseCommaSeparatedColumns("id, name,age", "--cols")); + ASSERT_EQ(parsed.size(), 3U); + ASSERT_EQ(parsed[0], "id"); + ASSERT_EQ(parsed[1], "name"); + ASSERT_EQ(parsed[2], "age"); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedColumnsRejectsInvalidInput) { + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("", "--cols")); + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("id,", "--cols")); + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("id,,name", "--cols")); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsWorks) { + ASSERT_OK_AND_ASSIGN( + auto parsed, paimon::benchmark::ParseDelimitedOptions("k1:v1;k2:v2", "--paimon_option")); + ASSERT_EQ(parsed.size(), 2U); + ASSERT_EQ(parsed[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(parsed[1], std::make_pair(std::string("k2"), std::string("v2"))); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsTrimsKeyAndValue) { + ASSERT_OK_AND_ASSIGN(auto parsed, paimon::benchmark::ParseDelimitedOptions(" k1 : v1 ; k2: v2 ", + "--paimon_option")); + ASSERT_EQ(parsed.size(), 2U); + ASSERT_EQ(parsed[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(parsed[1], std::make_pair(std::string("k2"), std::string("v2"))); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsRejectsInvalidInput) { + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1:v1;", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1:", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions(":v1", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1: ", "--paimon_option")); +} + +TEST(CliOptionParsingTest, ParseStringOptionArgWorksForEqualsAndSeparatedForms) { + { + ArgvHolder argv_holder({"prog", "--foo=bar"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 1); + ASSERT_EQ(value, "bar"); + } + + { + ArgvHolder argv_holder({"prog", "--foo", "bar"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(value, "bar"); + } +} + +TEST(CliOptionParsingTest, ParseStringOptionArgRejectsMissingValue) { + ArgvHolder argv_holder({"prog", "--foo"}); + int32_t arg_index = 1; + std::string value; + ASSERT_NOK(paimon::benchmark::ParseStringOptionArg(argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", + &arg_index, &value)); +} + +TEST(CliOptionParsingTest, ParseStringOptionArgIgnoresOtherOptions) { + ArgvHolder argv_holder({"prog", "--bar=baz"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_FALSE(is_parsed); + ASSERT_EQ(arg_index, 1); + ASSERT_TRUE(value.empty()); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedOptionArgAndDelimitedRepeatableOptionArgWorks) { + { + ArgvHolder argv_holder({"prog", "--cols", "id,name"}); + int32_t arg_index = 1; + std::vector columns; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseCommaSeparatedOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--cols", &arg_index, &columns)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(columns.size(), 2U); + ASSERT_EQ(columns[0], "id"); + ASSERT_EQ(columns[1], "name"); + } + + { + ArgvHolder argv_holder({"prog", "--paimon_option", "k1:v1;k2:v2"}); + int32_t arg_index = 1; + std::vector> options; + ASSERT_OK_AND_ASSIGN(bool is_parsed, paimon::benchmark::ParseDelimitedRepeatableOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--paimon_option", + &arg_index, &options)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(options.size(), 2U); + ASSERT_EQ(options[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(options[1], std::make_pair(std::string("k2"), std::string("v2"))); + } +} + +} // namespace +} // namespace paimon::testing diff --git a/benchmark/read_write_benchmark.cpp b/benchmark/read_write_benchmark.cpp new file mode 100644 index 00000000..5cd7ffb5 --- /dev/null +++ b/benchmark/read_write_benchmark.cpp @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "benchmark/benchmark.h" +#include "benchmark/benchmark_suite.h" + +int main(int argc, char** argv) { + if (paimon::benchmark::HasHelpFlag(static_cast(argc), argv)) { + paimon::benchmark::PrintPaimonBenchmarkCliHelp(); + return 0; + } + + const paimon::Status parse_status = paimon::benchmark::ParsePaimonBenchmarkCliArgs(&argc, argv); + if (!parse_status.ok()) { + std::cerr << "paimon-read-write-benchmark: " << parse_status.ToString() << std::endl; + std::cerr << "Try 'paimon-read-write-benchmark --help' for more information." << std::endl; + return 1; + } + + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/build_support/asan_symbolize.py b/build_support/asan_symbolize.py index bffb75a1..854090ae 100755 --- a/build_support/asan_symbolize.py +++ b/build_support/asan_symbolize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #===- lib/asan/scripts/asan_symbolize.py -----------------------------------===# # # The LLVM Compiler Infrastructure diff --git a/build_support/iwyu/iwyu.sh b/build_support/iwyu/iwyu.sh index 09b48039..7fb16f03 100755 --- a/build_support/iwyu/iwyu.sh +++ b/build_support/iwyu/iwyu.sh @@ -47,7 +47,7 @@ affected_files() { include-what-you-use --version if [[ "${1:-}" == "all" ]]; then - python $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \ -- $IWYU_ARGS #| awk -f $ROOT/build_support/iwyu/iwyu-filter.awk elif [[ "${1:-}" == "match" ]]; then @@ -60,7 +60,7 @@ elif [[ "${1:-}" == "match" ]]; then done echo "Running IWYU on $IWYU_FILE_LIST" - python $ROOT/build_support/iwyu/iwyu_tool.py \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \ -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST -- \ $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk else @@ -77,7 +77,7 @@ else IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p" done - python $ROOT/build_support/iwyu/iwyu_tool.py \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \ -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST -- \ $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk > $IWYU_LOG fi diff --git a/build_support/iwyu/iwyu_tool.py b/build_support/iwyu/iwyu_tool.py index 1429e0c0..87cf0307 100755 --- a/build_support/iwyu/iwyu_tool.py +++ b/build_support/iwyu/iwyu_tool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This file has been imported into the apache source tree from # the IWYU source tree as of version 0.8 @@ -65,7 +65,7 @@ -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -G Ninja ... - $ python iwyu_tool.py -p . + $ python3 iwyu_tool.py -p . See iwyu_tool.py -h for more details on command-line arguments. """ diff --git a/build_support/run_clang_format.py b/build_support/run_clang_format.py index 5b632060..fd653a53 100755 --- a/build_support/run_clang_format.py +++ b/build_support/run_clang_format.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/build_support/run_clang_tidy.py b/build_support/run_clang_tidy.py index 611a7e21..1cc1b216 100755 --- a/build_support/run_clang_tidy.py +++ b/build_support/run_clang_tidy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index 13a83393..a5b1d655 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -14,3 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library. +race:liblumina.so +race:libjindosdk_c.so.6 diff --git a/build_support/ubsan-suppressions.txt b/build_support/ubsan-suppressions.txt index 13a83393..d3151a8a 100644 --- a/build_support/ubsan-suppressions.txt +++ b/build_support/ubsan-suppressions.txt @@ -14,3 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# Arrow's ISO8601 string-to-timestamp parser intentionally reaches the int64 +# nanosecond boundary for values such as 1677-09-21 00:12:43.145224192. +signed-integer-overflow:std::chrono::__duration_cast_impl diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index ecc83a75..3df51435 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -17,11 +17,92 @@ set -eux -source_dir=${1} -enable_sanitizer=${2:-false} -check_clang_tidy=${3:-false} -build_type=${4:-Debug} -install_smoke=${5:-false} +usage() { + echo "Usage: $0 --source_dir [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type ] [--lint_git_target_commit ] [--install_smoke]" +} + +source_dir="" +enable_asan="false" +enable_ubsan="false" +enable_tsan="false" +check_clang_tidy="false" +build_type="Debug" +lint_git_target_commit="origin/main" +install_smoke="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --source_dir) + if [[ $# -lt 2 ]]; then + echo "Missing value for --source_dir" >&2 + usage >&2 + exit 1 + fi + source_dir=$2 + shift 2 + ;; + --enable_asan) + enable_asan="true" + shift + ;; + --enable_ubsan) + enable_ubsan="true" + shift + ;; + --enable_tsan) + enable_tsan="true" + shift + ;; + --check_clang_tidy) + check_clang_tidy="true" + shift + ;; + --build_type) + if [[ $# -lt 2 ]]; then + echo "Missing value for --build_type" >&2 + usage >&2 + exit 1 + fi + build_type=$2 + shift 2 + ;; + --lint_git_target_commit) + if [[ $# -lt 2 ]]; then + echo "Missing value for --lint_git_target_commit" >&2 + usage >&2 + exit 1 + fi + lint_git_target_commit=$2 + shift 2 + ;; + --install_smoke) + install_smoke="true" + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${source_dir}" ]]; then + echo "--source_dir is required" >&2 + usage >&2 + exit 1 +fi + +if [[ "${enable_asan}" == "true" && "${enable_tsan}" == "true" ]]; then + echo "ASAN and TSAN cannot be enabled together" >&2 + usage >&2 + exit 1 +fi + build_dir="${source_dir}/build" if [[ -n "${PAIMON_BUILD_JOBS:-}" ]]; then @@ -47,8 +128,13 @@ mkdir -p "${build_dir}" pushd "${build_dir}" ENABLE_LUMINA="ON" +ENABLE_TANTIVY="ON" if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then ENABLE_LUMINA="OFF" + ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image. +fi +if [[ "${enable_tsan}" == "true" ]]; then + ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented. fi CMAKE_ARGS=( @@ -56,15 +142,21 @@ CMAKE_ARGS=( "-DCMAKE_BUILD_TYPE=${build_type}" "-DPAIMON_BUILD_TESTS=ON" "-DPAIMON_ENABLE_JINDO=ON" + "-DPAIMON_ENABLE_S3=ON" "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" + "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) -if [[ "${enable_sanitizer}" == "true" ]]; then - CMAKE_ARGS+=( - "-DPAIMON_USE_ASAN=ON" - "-DPAIMON_USE_UBSAN=ON" - ) +if [[ "${enable_asan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_ASAN=ON") +fi +if [[ "${enable_ubsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_UBSAN=ON") +fi +if [[ "${enable_tsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_TSAN=ON") fi cmake "${CMAKE_ARGS[@]}" "${source_dir}" diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh new file mode 100755 index 00000000..bdb8f622 --- /dev/null +++ b/ci/scripts/setup_rust.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Install the Rust toolchain + cbindgen required to build the +# tantivy-fts FFI crate (crates/tantivy_ffi) from CI. +# +# The dev container (see .devcontainer/) already has these preinstalled; +# this script is for the GitHub Actions runners. Called by +# .github/workflows/build_and_test.yaml before ci/scripts/build_paimon.sh. +# +# Idempotent: a second invocation is a no-op when the tools already exist. + +set -eux + +RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} +# 1.88.0 is the minimum required by transitive crates (e.g. time 0.3.47). +RUST_VERSION=${RUST_VERSION:-1.88.0} +CBINDGEN_VERSION=${CBINDGEN_VERSION:-0.29.2} + +# Install rustup + default toolchain if cargo isn't on PATH yet. +if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain "${RUST_VERSION}" --profile minimal --no-modify-path +fi + +# Export for the remainder of the CI job. +export PATH="${HOME}/.cargo/bin:${PATH}" +echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH:-/dev/null}" || true + +rustup toolchain install "${RUST_VERSION}" --profile minimal +rustup default "${RUST_VERSION}" +rustup component add rustfmt clippy + +# cbindgen is used by the crate's build.rs to emit the C header that the +# C++ side includes. Corrosion will also run cbindgen at CMake configure +# time; both paths need it available. +if ! command -v cbindgen >/dev/null 2>&1; then + cargo install cbindgen --version "${CBINDGEN_VERSION}" --locked +fi + +rustc --version +cargo --version +cbindgen --version diff --git a/cmake_modules/BuildAwsAuth.cmake b/cmake_modules/BuildAwsAuth.cmake new file mode 100644 index 00000000..62e749d9 --- /dev/null +++ b/cmake_modules/BuildAwsAuth.cmake @@ -0,0 +1,189 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(ExternalProject) +include(GNUInstallDirs) + +function(paimon_add_s2n_project) + if(DEFINED ENV{PAIMON_AWS_S2N_URL}) + set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}") + elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}") + set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}") + else() + set(S2N_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip" + ) + endif() + set(S2N_C_FLAGS "${EP_C_FLAGS}") + set(S2N_CXX_FLAGS "${EP_CXX_FLAGS}") + string(REPLACE "-Wdocumentation" "" S2N_C_FLAGS "${S2N_C_FLAGS}") + string(REPLACE "-Wdocumentation" "" S2N_CXX_FLAGS "${S2N_CXX_FLAGS}") + externalproject_add(s2n_ep + URL ${S2N_URL} + URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM} + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_C_FLAGS=${S2N_C_FLAGS} + -DCMAKE_CXX_FLAGS=${S2N_CXX_FLAGS} + -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX} + -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR} + -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX} + -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} + -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DS2N_INTERN_LIBCRYPTO=OFF + BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a + ${THIRDPARTY_LOG_OPTIONS}) +endfunction() + +function(paimon_add_aws_c_project + NAME + VERSION + CHECKSUM + DEPENDS) + string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}") + string(TOUPPER "${URL_NAME}" URL_NAME) + set(URL_VAR "PAIMON_${URL_NAME}_URL") + set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME") + if(DEFINED ENV{${URL_VAR}}) + set(URL "$ENV{${URL_VAR}}") + elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}") + set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}") + else() + set(URL + "${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz" + ) + endif() + set(AWS_PLATFORM_CMAKE_ARGS) + list(APPEND + AWS_PLATFORM_CMAKE_ARGS + -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} + -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY} + -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY} + -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} + -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) + if(NAME STREQUAL "aws-c-cal") + list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON) + endif() + if(APPLE) + list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON) + endif() + # aws-c-io's installed package unconditionally looks for s2n on Unix even + # when it was built with Apple's native TLS backend. + if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth")) + list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON) + endif() + externalproject_add(${NAME}_ep + URL ${URL} + URL_HASH SHA256=${CHECKSUM} + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX} + -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR} + -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DENABLE_TESTING=OFF + ${AWS_PLATFORM_CMAKE_ARGS} + DEPENDS ${DEPENDS} + BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a + ${THIRDPARTY_LOG_OPTIONS}) +endfunction() + +function(build_aws_auth) + set(AWS_AUTH_PREFIX + "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install" + PARENT_SCOPE) + set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install") + set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}") + set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}") + file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include") + file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}") + + find_package(OpenSSL REQUIRED) + + set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + paimon_add_s2n_project() + list(APPEND AWS_IO_DEPENDS s2n_ep) + endif() + + paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION} + ${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "") + paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION} + ${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM} + aws-c-common_ep) + paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION} + ${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep) + paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION} + ${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM} + aws-c-common_ep) + paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION} + ${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}") + paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION} + ${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM} + "aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep") + paimon_add_aws_c_project(aws-c-auth + ${PAIMON_AWS_C_AUTH_BUILD_VERSION} + ${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM} + "aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep" + ) + + find_package(Threads REQUIRED) + set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS}) + if(APPLE) + list(APPEND + AWS_AUTH_PLATFORM_LIBS + "-framework Security" + "-framework CoreFoundation" + "-framework Network") + endif() + set(AWS_AUTH_TLS_LIBS) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a") + endif() + set(AWS_AUTH_RUNTIME_LIBS + "${AWS_AUTH_LIB_DIR}/libaws-c-http.a" + "${AWS_AUTH_LIB_DIR}/libaws-c-io.a" + "${AWS_AUTH_LIB_DIR}/libaws-c-compression.a" + "${AWS_AUTH_LIB_DIR}/libaws-c-cal.a" + "${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a" + "${AWS_AUTH_LIB_DIR}/libaws-c-common.a" + ${AWS_AUTH_TLS_LIBS}) + if(NOT APPLE) + list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group") + list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group") + endif() + # Keep OpenSSL after the static AWS archives. In embedded builds CMake can + # resolve these targets to static archives, where link order matters. + list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}" + "${OPENSSL_CRYPTO_LIBRARY}") + add_library(aws_auth_minimal STATIC IMPORTED GLOBAL) + set_target_properties(aws_auth_minimal + PROPERTIES IMPORTED_LOCATION + "${AWS_AUTH_LIB_DIR}/libaws-c-auth.a" + INTERFACE_INCLUDE_DIRECTORIES + "${AWS_AUTH_PREFIX}/include" + INTERFACE_LINK_LIBRARIES + "${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}") + add_dependencies(aws_auth_minimal aws-c-auth_ep) + + set(AWS_AUTH_INCLUDE_DIR + "${AWS_AUTH_PREFIX}/include" + PARENT_SCOPE) + set(AWS_AUTH_LIB_DIR + "${AWS_AUTH_LIB_DIR}" + PARENT_SCOPE) +endfunction() diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index 5f93160e..c0a9e14f 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -18,6 +18,43 @@ # Borrowed the file from Apache Arrow: # https://github.com/apache/arrow/blob/apache-arrow-17.0.0/cpp/cmake_modules/BuildUtils.cmake +function(paimon_link_libraries_whole_archive OUT_VAR) + set(_paimon_whole_archive_libs) + if(APPLE) + foreach(_paimon_lib IN LISTS ARGN) + list(APPEND _paimon_whole_archive_libs + "-Wl,-force_load,$" ${_paimon_lib}) + endforeach() + else() + list(APPEND + _paimon_whole_archive_libs + "-Wl,--whole-archive" + ${ARGN} + "-Wl,--no-whole-archive") + endif() + set(${OUT_VAR} + ${_paimon_whole_archive_libs} + PARENT_SCOPE) +endfunction() + +function(paimon_link_libraries_no_as_needed OUT_VAR) + set(_paimon_link_libs) + foreach(_paimon_lib IN LISTS ARGN) + if(APPLE) + list(APPEND _paimon_link_libs ${_paimon_lib}) + else() + list(APPEND + _paimon_link_libs + "-Wl,--no-as-needed" + ${_paimon_lib} + "-Wl,--as-needed") + endif() + endforeach() + set(${OUT_VAR} + ${_paimon_link_libs} + PARENT_SCOPE) +endfunction() + function(add_paimon_lib LIB_NAME) set(options BUILD_SHARED BUILD_STATIC) set(one_value_args SHARED_LINK_FLAGS) @@ -55,11 +92,15 @@ function(add_paimon_lib LIB_NAME) # Generate a single "objlib" from all C++ modules and link # that "objlib" into each library kind, to avoid compiling twice add_library(${LIB_NAME}_objlib OBJECT ${ARG_SOURCES}) - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_link_libraries(${LIB_NAME}_objlib + PRIVATE "$") + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") target_compile_options(${LIB_NAME}_objlib PRIVATE -Wno-global-constructors) endif() # Necessary to make static linking into other shared libraries work properly set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) + target_link_libraries(${LIB_NAME}_objlib PUBLIC paimon_sanitizer_flags) if(ARG_DEPENDENCIES) # In static-only builds, some dependency names are still declared as # *_shared. Map them to *_static when the shared target is unavailable. @@ -142,12 +183,20 @@ function(add_paimon_lib LIB_NAME) target_link_libraries(${LIB_NAME}_shared PUBLIC "$") - target_link_options(${LIB_NAME}_shared - PRIVATE - -Wl,--exclude-libs,ALL - -Wl,-Bsymbolic - -Wl,-z,defs - -Wl,--gc-sections) + if(NOT APPLE) + set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic + -Wl,--gc-sections) + # -z defs (--no-undefined) rejects the __asan_*/__tsan_*/__ubsan_* symbols that + # sanitizer-instrumented shared libraries legitimately leave undefined + # (they are resolved at load time from the executable's sanitizer + # runtime). Only enforce it for non-sanitizer builds. + if(NOT PAIMON_USE_ASAN + AND NOT PAIMON_USE_TSAN + AND NOT PAIMON_USE_UBSAN) + list(APPEND SHARED_LINK_OPTIONS -Wl,-z,defs) + endif() + target_link_options(${LIB_NAME}_shared PRIVATE ${SHARED_LINK_OPTIONS}) + endif() install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL} EXPORT PaimonTargets @@ -294,10 +343,16 @@ function(add_test_case REL_TEST_NAME) add_dependencies(${TEST_NAME} ${ARG_EXTRA_DEPENDENCIES}) endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors) endif() target_compile_options(${TEST_NAME} PRIVATE -fno-access-control) + # Test sources initialize char / vector from raw byte values like + # {1, -1, ...}; char is unsigned by default on aarch64, which triggers + # -Wnarrowing. Disable it for tests so we don't have to sprinkle + # static_cast(-1) everywhere. Production code (src/paimon/...) keeps it. + target_compile_options(${TEST_NAME} PRIVATE -Wno-narrowing) add_test(${TEST_NAME} ${BUILD_SUPPORT_DIR}/run-test.sh @@ -368,3 +423,122 @@ function(add_paimon_test REL_TEST_NAME) ${PCH_ARGS} ${ARG_UNPARSED_ARGUMENTS}) endfunction() + +function(add_benchmark_case REL_BENCHMARK_NAME) + set(options ENABLED) + set(one_value_args) + set(multi_value_args + SOURCES + STATIC_LINK_LIBS + EXTRA_LINK_LIBS + EXTRA_INCLUDES + LABELS + PREFIX) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + if(NOT PAIMON_BUILD_BENCHMARKS AND NOT ARG_ENABLED) + return() + endif() + + get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE) + + if(ARG_PREFIX) + set(BENCHMARK_NAME "${ARG_PREFIX}-${BENCHMARK_NAME}") + endif() + + if(ARG_SOURCES) + set(SOURCES ${ARG_SOURCES}) + else() + set(SOURCES "${REL_BENCHMARK_NAME}.cpp") + endif() + + string(REPLACE "_" "-" BENCHMARK_NAME ${BENCHMARK_NAME}) + set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}") + message(STATUS ${BENCHMARK_NAME}) + add_executable(${BENCHMARK_NAME} ${SOURCES}) + + if(ARG_STATIC_LINK_LIBS) + target_link_libraries(${BENCHMARK_NAME} PRIVATE ${ARG_STATIC_LINK_LIBS}) + endif() + + if(ARG_EXTRA_LINK_LIBS) + target_link_libraries(${BENCHMARK_NAME} PRIVATE ${ARG_EXTRA_LINK_LIBS}) + endif() + + if(ARG_EXTRA_INCLUDES) + target_include_directories(${BENCHMARK_NAME} SYSTEM PUBLIC ${ARG_EXTRA_INCLUDES}) + endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") + target_compile_options(${BENCHMARK_NAME} PRIVATE -Wno-global-constructors) + endif() + target_compile_options(${BENCHMARK_NAME} PRIVATE -fno-access-control) + + add_test(${BENCHMARK_NAME} + ${BUILD_SUPPORT_DIR}/run-test.sh + ${CMAKE_BINARY_DIR} + benchmark + ${BENCHMARK_PATH}) + + foreach(TARGET ${ARG_LABELS}) + add_dependencies(${TARGET} ${BENCHMARK_NAME}) + endforeach() + + set(LABELS) + list(APPEND LABELS "benchmark") + if(ARG_LABELS) + list(APPEND LABELS ${ARG_LABELS}) + endif() + + foreach(LABEL ${ARG_LABELS}) + set(LABEL_BENCHMARK_NAME "benchmark-${LABEL}") + if(NOT TARGET ${LABEL_BENCHMARK_NAME}) + add_custom_target(${LABEL_BENCHMARK_NAME} + ctest -L "${LABEL}" --output-on-failure + USES_TERMINAL) + endif() + add_dependencies(${LABEL_BENCHMARK_NAME} ${BENCHMARK_NAME}) + endforeach() + + set_property(TEST ${BENCHMARK_NAME} + APPEND + PROPERTY LABELS ${LABELS}) +endfunction() + +function(add_paimon_benchmark REL_BENCHMARK_NAME) + set(options) + set(one_value_args PREFIX) + set(multi_value_args LABELS) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + + if(ARG_PREFIX) + set(PREFIX ${ARG_PREFIX}) + else() + set(PREFIX "paimon") + endif() + + if(ARG_LABELS) + set(LABELS ${ARG_LABELS}) + else() + set(LABELS "paimon-benchmarks") + endif() + + add_benchmark_case(${REL_BENCHMARK_NAME} + PREFIX + ${PREFIX} + LABELS + ${LABELS} + ${ARG_UNPARSED_ARGUMENTS}) +endfunction() diff --git a/cmake_modules/CorrosionFetch.cmake b/cmake_modules/CorrosionFetch.cmake new file mode 100644 index 00000000..b56b2795 --- /dev/null +++ b/cmake_modules/CorrosionFetch.cmake @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Pull Corrosion-rs via FetchContent so we can import crates/tantivy_ffi as +# CMake targets for the tantivy-fulltext global index. +# +# Pinned to v0.5.2 (stable release). Requires CMake >= 3.22. + +include(FetchContent) + +# Corrosion does heavy cargo/rustc work at configure+build time; pin tag for +# reproducibility and allow override via env var for offline builds. +set(PAIMON_CORROSION_TAG + "v0.5.2" + CACHE STRING "Git tag of corrosion-rs to fetch; change only when upgrading. v0.5.1+ + is required for rustup >= 1.28 whose `rustup toolchain list --verbose` + output format broke v0.5.0's FindRust.cmake regex.") + +set(PAIMON_CORROSION_REPO + "https://github.com/corrosion-rs/corrosion.git" + CACHE STRING "Override to a private mirror for offline / firewalled builds.") + +# Help Corrosion find rustc/cargo when CMake is invoked without a login shell +# or when rustup is installed to a non-default location. We try, in order: +# 1. Existing Rust_COMPILER cache variable (user override) +# 2. $CARGO_HOME/bin/rustc (when env var set) +# 3. $HOME/.cargo/bin/rustc (rustup's default install) +# 4. Fallback: let Corrosion's FindRust.cmake try its own detection +function(_paimon_find_rustup_bin _var _name) + if(DEFINED ENV{CARGO_HOME} AND EXISTS "$ENV{CARGO_HOME}/bin/${_name}") + set(${_var} + "$ENV{CARGO_HOME}/bin/${_name}" + PARENT_SCOPE) + elseif(DEFINED ENV{HOME} AND EXISTS "$ENV{HOME}/.cargo/bin/${_name}") + set(${_var} + "$ENV{HOME}/.cargo/bin/${_name}" + PARENT_SCOPE) + endif() +endfunction() + +if(NOT DEFINED Rust_COMPILER OR Rust_COMPILER STREQUAL "") + _paimon_find_rustup_bin(_rustc_path rustc) + if(_rustc_path) + set(Rust_COMPILER + "${_rustc_path}" + CACHE FILEPATH "rustc") + endif() +endif() +if(NOT DEFINED Rust_CARGO OR Rust_CARGO STREQUAL "") + _paimon_find_rustup_bin(_cargo_path cargo) + if(_cargo_path) + set(Rust_CARGO + "${_cargo_path}" + CACHE FILEPATH "cargo") + endif() +endif() +# Corrosion reads `rustup which rustc` to resolve the real toolchain binary. +# If CMake is invoked from a non-login shell, $PATH may miss ~/.cargo/bin and +# `rustup` can't be found. Prepend rustup's bin dir so child processes see it. +if(DEFINED Rust_COMPILER) + get_filename_component(_rustup_bin_dir "${Rust_COMPILER}" DIRECTORY) + if(_rustup_bin_dir AND NOT "$ENV{PATH}" MATCHES "${_rustup_bin_dir}") + set(ENV{PATH} "${_rustup_bin_dir}:$ENV{PATH}") + endif() +endif() +message(STATUS "Corrosion: Rust_COMPILER=${Rust_COMPILER}") +message(STATUS "Corrosion: Rust_CARGO=${Rust_CARGO}") + +fetchcontent_declare(Corrosion + GIT_REPOSITORY "${PAIMON_CORROSION_REPO}" + GIT_TAG "${PAIMON_CORROSION_TAG}" + GIT_SHALLOW TRUE) +fetchcontent_makeavailable(Corrosion) diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake index 45cf4476..4aacc33d 100644 --- a/cmake_modules/DefineOptions.cmake +++ b/cmake_modules/DefineOptions.cmake @@ -98,11 +98,21 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(PAIMON_USE_CCACHE "Use ccache when compiling (if available)" ON) + define_option(PAIMON_USE_APPLE_LIBCXX_WITH_CLANG + "Use Apple SDK libc++ headers when building with upstream Clang on macOS" + ON) + #---------------------------------------------------------------------- set_option_category("Test") define_option(PAIMON_BUILD_TESTS "Build the Paimon googletest unit tests" OFF) + define_option(PAIMON_ENABLE_NETWORK_TESTS + "Enable tests that access real remote services over the network" OFF) + + define_option(PAIMON_BUILD_BENCHMARKS + "Build the Paimon Google Benchmark performance benchmarks" OFF) + if(PAIMON_BUILD_SHARED) set(PAIMON_TEST_LINKAGE_DEFAULT "shared") else() @@ -237,6 +247,13 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") AUTO BUNDLED SYSTEM) + + define_option_string(benchmark_SOURCE + "Dependency source for Google Benchmark" + "" + AUTO + BUNDLED + SYSTEM) endif() macro(validate_config) diff --git a/cmake_modules/FindbenchmarkAlt.cmake b/cmake_modules/FindbenchmarkAlt.cmake new file mode 100644 index 00000000..f3eeadf2 --- /dev/null +++ b/cmake_modules/FindbenchmarkAlt.cmake @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(_PAIMON_BENCHMARK_ROOTS ${benchmark_ROOT} ${BENCHMARK_ROOT} ${PAIMON_PACKAGE_PREFIX}) +list(REMOVE_ITEM _PAIMON_BENCHMARK_ROOTS "") +if(_PAIMON_BENCHMARK_ROOTS) + set(_PAIMON_BENCHMARK_FIND_ARGS HINTS ${_PAIMON_BENCHMARK_ROOTS} NO_DEFAULT_PATH) +endif() + +find_package(benchmark CONFIG QUIET ${_PAIMON_BENCHMARK_FIND_ARGS}) + +if(NOT TARGET benchmark::benchmark) + find_path(BENCHMARK_INCLUDE_DIR + NAMES benchmark/benchmark.h ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES include) + find_library(BENCHMARK_LIBRARY + NAMES benchmark ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES lib lib64) + find_library(BENCHMARK_MAIN_LIBRARY + NAMES benchmark_main ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES lib lib64) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(benchmarkAlt REQUIRED_VARS BENCHMARK_INCLUDE_DIR + BENCHMARK_LIBRARY) + + if(benchmarkAlt_FOUND) + if(NOT TARGET benchmark::benchmark) + add_library(benchmark::benchmark UNKNOWN IMPORTED) + set_target_properties(benchmark::benchmark + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + endif() + + if(BENCHMARK_MAIN_LIBRARY AND NOT TARGET benchmark::benchmark_main) + add_library(benchmark::benchmark_main UNKNOWN IMPORTED) + set_target_properties(benchmark::benchmark_main + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_MAIN_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + endif() + endif() +else() + set(benchmarkAlt_FOUND TRUE) +endif() + +unset(_PAIMON_BENCHMARK_ROOTS) +unset(_PAIMON_BENCHMARK_FIND_ARGS) diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake index 043c043b..4df2f560 100644 --- a/cmake_modules/SetupCxxFlags.cmake +++ b/cmake_modules/SetupCxxFlags.cmake @@ -196,6 +196,24 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STRE # the default standard library which does not support C++11. libc++ is the # default from 10.9 onward. set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++") + elseif(APPLE AND PAIMON_USE_APPLE_LIBCXX_WITH_CLANG) + execute_process(COMMAND xcrun --show-sdk-path + OUTPUT_VARIABLE PAIMON_MACOS_SDK_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(NOT PAIMON_MACOS_SDK_PATH) + message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but xcrun could not find the macOS SDK" + ) + endif() + set(PAIMON_MACOS_LIBCXX_INCLUDE_DIR "${PAIMON_MACOS_SDK_PATH}/usr/include/c++/v1") + if(NOT EXISTS "${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}/cstdlib") + message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but libc++ headers were not found at ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}" + ) + endif() + message(STATUS "Using Apple libc++ headers with Clang: ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}" + ) + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR} -include cstdlib" + ) endif() endif() diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index ec93aa50..32d7b1d1 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -23,6 +23,8 @@ set(THIRDPARTY_LOG_OPTIONS LOG_INSTALL 1 LOG_DOWNLOAD + 1 + LOG_OUTPUT_ON_FAILURE 1) set(THIRDPARTY_CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}") if(CMAKE_GENERATOR_TOOLSET) @@ -245,6 +247,18 @@ else() endif() endif() +if(DEFINED ENV{PAIMON_BENCHMARK_URL}) + set(BENCHMARK_SOURCE_URL "$ENV{PAIMON_BENCHMARK_URL}") +else() + if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_BENCHMARK_PKG_NAME}") + set_urls(BENCHMARK_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_BENCHMARK_PKG_NAME}") + else() + set_urls(BENCHMARK_SOURCE_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz" + ) + endif() +endif() + if(DEFINED ENV{PAIMON_TBB_URL}) set(TBB_SOURCE_URL "$ENV{PAIMON_TBB_URL}") else() @@ -364,7 +378,7 @@ if(NOT MSVC_TOOLCHAIN) # Set -fPIC on all external projects string(APPEND EP_CXX_FLAGS " -fPIC -Wno-error -Wno-sign-compare -Wno-ignored-attributes") - string(APPEND EP_C_FLAGS " -fPIC") + string(APPEND EP_C_FLAGS " -fPIC -Wno-error") endif() if(PAIMON_USE_CXX11_ABI) @@ -557,6 +571,8 @@ function(paimon_get_dependency_compat_target DEPENDENCY_NAME OUT_VAR) set(_target libprotobuf) elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") set(_target GTest::gtest) + elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark") + set(_target benchmark::benchmark) elseif("${DEPENDENCY_NAME}" STREQUAL "RE2") set(_target re2::re2) elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy") @@ -643,6 +659,8 @@ macro(paimon_build_dependency DEPENDENCY_NAME) build_avro() elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") build_gtest() + elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark") + build_benchmark() else() message(FATAL_ERROR "No bundled build rule for ${DEPENDENCY_NAME}") endif() @@ -753,19 +771,12 @@ macro(build_lucene) set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install") - set(LUCENE_CMAKE_CXX_FLAGS "-pthread") - if(PAIMON_USE_CXX11_ABI) - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") - else() - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") - endif() - set(LUCENE_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DLUCENE_BUILD_SHARED=OFF" "-DENABLE_TEST=OFF" - "-DCMAKE_C_FLAGS=-pthread" - "-DCMAKE_CXX_FLAGS=${LUCENE_CMAKE_CXX_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS} -pthread" + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -pthread" "-DCMAKE_EXE_LINKER_FLAGS=-pthread" "-DBoost_NO_BOOST_CMAKE=ON" "-DBoost_NO_SYSTEM_PATHS=ON" @@ -773,6 +784,11 @@ macro(build_lucene) "-DBoost_INCLUDE_DIR=${BOOST_INCLUDE_DIR}" "-DBoost_LIBRARY_DIR=${BOOST_LIBRARY_DIR}" "-DBOOST_ROOT=${BOOST_INSTALL}" + # Force FindBoost module mode only; ignore system BoostConfig.cmake and + # system library paths so lucene_ep links against our vendored boost 1.66, + # not a system-installed newer version (e.g. 1.83) with ABI differences. + "-DBoost_NO_BOOST_CMAKE=ON" + "-DBoost_NO_SYSTEM_PATHS=ON" "-DBoost_CHRONO_FOUND=TRUE" "-DBoost_THREAD_FOUND=TRUE" "-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIR}" @@ -818,7 +834,7 @@ macro(build_lucene) boost_chrono boost_atomic pthread - dl) + ${CMAKE_DL_LIBS}) add_dependencies(lucene lucene_ep) endmacro() @@ -828,6 +844,8 @@ macro(build_jieba) externalproject_add(limonp_ep URL ${LIMONP_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "") message(STATUS "Building jieba from source") @@ -904,17 +922,9 @@ macro(build_fmt) "${FMT_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${FMT_STATIC_LIB_NAME}${FMT_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(FMT_LIBRARIES ${FMT_STATIC_LIB}) - set(FMT_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(FMT_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" FMT_CMAKE_CXX_FLAGS ${FMT_CMAKE_CXX_FLAGS}) - set(FMT_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} - "-DCMAKE_CXX_FLAGS=${FMT_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${FMT_CMAKE_C_FLAGS}" - -DFMT_TEST=OFF - -DFMT_DOC=OFF) + set(FMT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} + -DFMT_TEST=OFF -DFMT_DOC=OFF) set(FMT_CONFIGURE CMAKE_ARGS ${FMT_CMAKE_ARGS}) externalproject_add(fmt_ep URL ${FMT_SOURCE_URL} @@ -950,12 +960,16 @@ macro(build_boost) ${BOOST_LIBRARY_DIR}/libboost_chrono.a ${BOOST_LIBRARY_DIR}/libboost_iostreams.a) - set(BOOST_CXX_FLAGS "-fPIC") - if(PAIMON_USE_CXX11_ABI) - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(BOOST_TOOLSET clang) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(BOOST_TOOLSET gcc) else() - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") + message(FATAL_ERROR "Unsupported compiler for Boost: ${CMAKE_CXX_COMPILER_ID}") endif() + set(BOOST_USER_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/boost-user-config.jam") + file(WRITE ${BOOST_USER_CONFIG} + "using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n") externalproject_add(boost_ep URL ${BOOST_SOURCE_URL} @@ -969,14 +983,17 @@ macro(build_boost) -sZLIB_INCLUDE=${ZLIB_INCLUDE_DIR} -sZLIB_LIBRARY_PATH=${ZLIB_PREFIX}/lib runtime-link=shared threading=multi variant=release - cxxflags=${BOOST_CXX_FLAGS} install + --user-config=${BOOST_USER_CONFIG} + toolset=${BOOST_TOOLSET} cxxflags=${EP_CXX_FLAGS} + linkflags=${EP_CXX_FLAGS} install INSTALL_COMMAND bash -c "mkdir -p ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/include/boost/* ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/*/include/boost/* ${BOOST_INSTALL}/include/boost" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} LOG_DOWNLOAD ON LOG_CONFIGURE ON - LOG_BUILD ON) + LOG_BUILD ON + LOG_OUTPUT_ON_FAILURE ON) include_directories(SYSTEM ${BOOST_INCLUDE_DIR}) @@ -1120,17 +1137,9 @@ macro(build_zstd) "${ZSTD_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(ZSTD_LIBRARIES ${ZSTD_STATIC_LIB}) - set(ZSTD_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ZSTD_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ZSTD_CMAKE_CXX_FLAGS ${ZSTD_CMAKE_CXX_FLAGS}) - set(ZSTD_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} - "-DCMAKE_CXX_FLAGS=${ZSTD_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ZSTD_CMAKE_C_FLAGS}" - -DZSTD_BUILD_SHARED=OFF - -DZSTD_BUILD_PROGRAMS=OFF) + set(ZSTD_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} + -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_PROGRAMS=OFF) set(ZSTD_CONFIGURE SOURCE_SUBDIR "build/cmake" CMAKE_ARGS ${ZSTD_CMAKE_ARGS}) externalproject_add(zstd_ep @@ -1270,14 +1279,8 @@ macro(build_jindosdk_nextarch) get_target_property(JINDOSDK_C_LIBRARY_LOCATION jindosdk::c_sdk IMPORTED_LOCATION) get_filename_component(JINDOSDK_C_DIR_ROOT "${JINDOSDK_C_INCLUDE_DIR}" DIRECTORY) - # Compile flags for jindosdk-nextarch - set(JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - set(JINDOSDK_NEXTARCH_CMAKE_C_FLAGS "${EP_C_FLAGS}") set(JINDOSDK_NEXTARCH_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" - "-DCMAKE_CXX_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_C_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" -DJINDOSDK_ROOT=${JINDOSDK_C_DIR_ROOT} -DJINDOSDK_LIBRARY_NAME=${JINDOSDK_C_DYNAMIC_LIB_NAME}) @@ -1296,7 +1299,8 @@ macro(build_jindosdk_nextarch) PROPERTIES IMPORTED_LOCATION "${JINDOSDK_NEXTARCH_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${JINDOSDK_NEXTARCH_INCLUDE_DIR}") - target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread dl) + target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread + ${CMAKE_DL_LIBS}) list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_NEXTARCH_INCLUDE_DIR}) add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep) @@ -1318,6 +1322,26 @@ macro(build_protobuf) get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}" DIRECTORY) + get_target_property(THIRDPARTY_ZLIB_LIBRARY zlib IMPORTED_LOCATION) + set(PROTOBUF_ZLIB_LIBRARY_ARGS) + foreach(_PAIMON_ZLIB_LOCATION_PROPERTY + IMPORTED_LOCATION + IMPORTED_LOCATION_NOCONFIG + IMPORTED_LOCATION_RELEASE + IMPORTED_LOCATION_DEBUG + IMPORTED_LOCATION_RELWITHDEBINFO + IMPORTED_LOCATION_MINSIZEREL) + if(NOT THIRDPARTY_ZLIB_LIBRARY AND TARGET ZLIB::ZLIB) + get_target_property(THIRDPARTY_ZLIB_LIBRARY ZLIB::ZLIB + ${_PAIMON_ZLIB_LOCATION_PROPERTY}) + endif() + endforeach() + unset(_PAIMON_ZLIB_LOCATION_PROPERTY) + if(THIRDPARTY_ZLIB_LIBRARY) + set(PROTOBUF_ZLIB_LIBRARY_ARGS + "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_LIBRARY}" + "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_LIBRARY}") + endif() # Strip lto flags (which may be added by dh_auto_configure) # See https://github.com/protocolbuffers/protobuf/issues/7092 @@ -1337,6 +1361,7 @@ macro(build_protobuf) "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX=) + list(APPEND PROTOBUF_CMAKE_ARGS ${PROTOBUF_ZLIB_LIBRARY_ARGS}) set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS}) externalproject_add(protobuf_ep @@ -1386,14 +1411,9 @@ macro(build_avro) get_target_property(AVRO_FMT_INCLUDE_DIR fmt INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(AVRO_FMT_ROOT "${AVRO_FMT_INCLUDE_DIR}" DIRECTORY) - set(AVRO_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(AVRO_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - set(AVRO_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${AVRO_PREFIX}" - "-DCMAKE_CXX_FLAGS=${AVRO_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${AVRO_CMAKE_C_FLAGS}" "-DAVRO_BUILD_TESTS=OFF" "-DAVRO_BUILD_EXECUTABLES=OFF" "-DZLIB_ROOT=${AVRO_ZLIB_ROOT}" @@ -1441,13 +1461,14 @@ macro(build_orc) message(STATUS "PAIMON_RPATH value: ${PAIMON_RPATH}") set(ORC_RPATH ${PAIMON_RPATH}) message(STATUS "ORC_RPATH value: ${ORC_RPATH}") - - string(REPLACE "-Werror" "" EP_CXX_FLAGS ${EP_CXX_FLAGS}) - - set(ORC_CMAKE_CXX_FLAGS - "${EP_CXX_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") - set(ORC_CMAKE_C_FLAGS - "${EP_C_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") + set(ORC_LINKER_FLAGS) + if(NOT "${ORC_RPATH}" STREQUAL "") + list(APPEND + ORC_LINKER_FLAGS + "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}") + endif() set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-prefix") set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") @@ -1456,19 +1477,10 @@ macro(build_orc) set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/liborc.a") - message("ORC_STATIC_LIB IS ${ORC_STATIC_LIB}") - message("ORC_CMAKE_CXX_FLAGS ${ORC_CMAKE_CXX_FLAGS}") - message("ORC_CMAKE_C_FLAGS ${ORC_CMAKE_C_FLAGS}") - set(ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}" - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + ${ORC_LINKER_FLAGS} "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" "-DLZ4_HOME=${ORC_LZ4_ROOT}" "-DZSTD_HOME=${ORC_ZSTD_ROOT}" @@ -1533,9 +1545,7 @@ macro(build_arrow) get_target_property(ARROW_RE2_INCLUDE_DIR re2::re2 INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(ARROW_RE2_ROOT "${ARROW_RE2_INCLUDE_DIR}" DIRECTORY) - set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ARROW_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ARROW_CMAKE_CXX_FLAGS ${ARROW_CMAKE_CXX_FLAGS}) + set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") # Fix for thrift Mutex.h missing #include (GCC 15 strictness) # Use -include to force include cstdint for all C++ files string(APPEND ARROW_CMAKE_CXX_FLAGS " -include cstdint") @@ -1570,8 +1580,7 @@ macro(build_arrow) ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}" "-DCMAKE_CXX_FLAGS=${ARROW_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" + -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_BUILD_SHARED=OFF -DARROW_BUILD_STATIC=ON @@ -1595,6 +1604,12 @@ macro(build_arrow) -DARROW_WITH_ZSTD=ON -DARROW_WITH_BZ2=OFF -DARROW_WITH_BROTLI=ON + -Dzstd_SOURCE=SYSTEM + -DSnappy_SOURCE=SYSTEM + -Dlz4_SOURCE=SYSTEM + -DZLIB_SOURCE=SYSTEM + -Dre2_SOURCE=SYSTEM + -Dzstd_ROOT=${ARROW_ZSTD_ROOT} -DZSTD_ROOT=${ARROW_ZSTD_ROOT} -DZLIB_ROOT=${ARROW_ZLIB_ROOT} -DSnappy_ROOT=${ARROW_SNAPPY_ROOT} @@ -1626,40 +1641,30 @@ macro(build_arrow) add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_dataset STATIC IMPORTED) set_target_properties(arrow_dataset PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_dataset.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_acero STATIC IMPORTED) set_target_properties(arrow_acero PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_acero.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(parquet STATIC IMPORTED) set_target_properties(parquet PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libparquet.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_bundled_dependencies STATIC IMPORTED) set_target_properties(arrow_bundled_dependencies PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_bundled_dependencies.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_dependencies(arrow arrow_ep) add_dependencies(parquet arrow_ep) @@ -1671,13 +1676,19 @@ macro(build_arrow) target_link_libraries(arrow_dataset INTERFACE arrow_acero) + # libarrow.a calls dlsym; keep ${CMAKE_DL_LIBS} in the interface so -ldl is placed + # after libarrow.a on linkers that resolve symbols strictly left-to-right. + # Every library that uses dl itself (arrow here; also lucene and jindosdk::nextarch) + # declares it on its own interface the same way; consumers inherit it transitively + # and must not list ${CMAKE_DL_LIBS} again themselves. target_link_libraries(arrow INTERFACE zstd snappy lz4 zlib re2::re2 - arrow_bundled_dependencies) + arrow_bundled_dependencies + ${CMAKE_DL_LIBS}) target_link_libraries(parquet INTERFACE zstd @@ -1692,9 +1703,6 @@ endmacro(build_arrow) macro(build_gtest) message(STATUS "Building gtest from source") - set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS}) - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-install") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -1715,10 +1723,7 @@ macro(build_gtest) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main.a") endif() set(GTEST_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" - "-DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GTEST_CMAKE_CXX_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" "-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${_GTEST_RUNTIME_DIR}" "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}=${_GTEST_RUNTIME_DIR}") @@ -1757,9 +1762,8 @@ endmacro() macro(build_tbb) message(STATUS "Building Tbb from source") - set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) + set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") + set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS}") string(REPLACE "-Wdocumentation" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_C_FLAGS ${TBB_CMAKE_C_FLAGS}) @@ -1782,7 +1786,6 @@ macro(build_tbb) "-DCMAKE_INSTALL_PREFIX=${TBB_PREFIX}" "-DCMAKE_CXX_FLAGS=${TBB_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${TBB_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${TBB_CMAKE_CXX_FLAGS}" -DTBB_TEST=OFF) externalproject_add(tbb_ep @@ -1794,15 +1797,58 @@ macro(build_tbb) add_library(tbb STATIC IMPORTED) set_target_properties(tbb PROPERTIES IMPORTED_LOCATION "${TBB_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${TBB_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${TBB_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${TBB_INCLUDE_DIR}") add_dependencies(tbb tbb_ep) endmacro(build_tbb) +macro(build_benchmark) + message(STATUS "Building benchmark from source") + + set(BENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/benchmark_ep-install") + set(BENCHMARK_INCLUDE_DIR "${BENCHMARK_PREFIX}/include") + set(BENCHMARK_STATIC_LIB + "${BENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BENCHMARK_MAIN_STATIC_LIB + "${BENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark_main${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(BENCHMARK_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BENCHMARK_PREFIX}" + -DBENCHMARK_ENABLE_TESTING=OFF + -DBENCHMARK_ENABLE_GTEST_TESTS=OFF + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=OFF) + + externalproject_add(benchmark_ep + URL ${BENCHMARK_SOURCE_URL} + URL_HASH "SHA256=${PAIMON_BENCHMARK_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${BENCHMARK_CMAKE_ARGS} + BUILD_BYPRODUCTS "${BENCHMARK_STATIC_LIB}" + "${BENCHMARK_MAIN_STATIC_LIB}") + + file(MAKE_DIRECTORY "${BENCHMARK_INCLUDE_DIR}") + + add_library(benchmark::benchmark STATIC IMPORTED) + set_target_properties(benchmark::benchmark + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + add_dependencies(benchmark::benchmark benchmark_ep) + + add_library(benchmark::benchmark_main STATIC IMPORTED) + set_target_properties(benchmark::benchmark_main + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + add_dependencies(benchmark::benchmark_main benchmark_ep) +endmacro() + macro(build_glog) message(STATUS "Building glog from source") + find_library(LIBUNWIND_LIBRARY NAMES unwind) + set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-install") set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include") if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") @@ -1811,20 +1857,12 @@ macro(build_glog) set(GLOG_LIB_SUFFIX "") endif() set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog${GLOG_LIB_SUFFIX}.a") - set(GLOG_CMAKE_CXX_FLAGS " -Wno-error ${EP_CXX_FLAGS}") - set(GLOG_CMAKE_C_FLAGS " -Wno-error ${EP_C_FLAGS}") - if(CMAKE_THREAD_LIBS_INIT) - string(APPEND GLOG_CMAKE_CXX_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - string(APPEND GLOG_CMAKE_C_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - endif() - set(GLOG_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} - -DWITH_GFLAGS=OFF - -DWITH_GTEST=OFF - -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS=${GLOG_CMAKE_C_FLAGS}) + set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} + -DWITH_GFLAGS=OFF -DWITH_GTEST=OFF) + if(NOT LIBUNWIND_LIBRARY) + list(APPEND GLOG_CMAKE_ARGS -DWITH_UNWIND=none) + endif() externalproject_add(glog_ep URL ${GLOG_SOURCE_URL} @@ -1842,7 +1880,6 @@ macro(build_glog) add_dependencies(glog glog_ep) - find_library(LIBUNWIND_LIBRARY NAMES unwind) if(LIBUNWIND_LIBRARY) target_link_libraries(glog INTERFACE ${LIBUNWIND_LIBRARY}) endif() @@ -1869,15 +1906,26 @@ if(PAIMON_ENABLE_ORC) resolve_dependency(Protobuf) resolve_dependency(ORC) endif() +if(PAIMON_BUILD_BENCHMARKS) + resolve_dependency(benchmark) +endif() if(PAIMON_ENABLE_JINDO) build_jindosdk_c() build_jindosdk_nextarch() endif() +if(PAIMON_ENABLE_S3) + include(BuildAwsAuth) + build_aws_auth() +endif() if(PAIMON_ENABLE_LUMINA) build_lumina() endif() if(PAIMON_ENABLE_LUCENE) build_boost() build_lucene() +endif() +# jieba (dict + headers) is needed by BOTH lucene-fts and the tantivy jieba +# tokenizer; build it whenever either backend is on, not only under lucene. +if(PAIMON_ENABLE_LUCENE OR PAIMON_ENABLE_TANTIVY) build_jieba() endif() diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index e539d1f8..c8aa0dd7 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -196,6 +196,210 @@ index 4d3acb491e..3906ff3c59 100644 int64_t pagesize_; ParquetDataPageVersion parquet_data_page_version_; ParquetVersion::type parquet_version_; + +--- a/cpp/src/parquet/file_reader.h ++++ b/cpp/src/parquet/file_reader.h +@@ -210,6 +210,17 @@ + ::arrow::Future<> WhenBuffered(const std::vector& row_groups, + const std::vector& column_indices) const; + ++ /// Pre-buffer arbitrary byte ranges (e.g., page-level ranges from OffsetIndex). ++ /// Unlike PreBuffer(), this does NOT set the column bitmap, so ++ /// GetColumnPageReader will use CachedInputStream (page-level cache path). ++ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options); ++ ++ /// Wait for arbitrary byte ranges to be pre-buffered. ++ ::arrow::Future<> WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const; ++ + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; + +--- a/cpp/src/parquet/file_reader.cc ++++ b/cpp/src/parquet/file_reader.cc +@@ -207,6 +207,117 @@ + return {col_start, col_length}; + } + ++// CachedInputStream: InputStream adapter that reads through ReadRangeCache with ++// zero-cost skip for non-cached pages. Used for page-level caching where only ++// specific pages are pre-buffered. ++// ++// Key behavior: ++// - Read(): On cache hit, returns cached data. On cache miss, returns zero-filled ++// buffer (zero I/O). This makes InputStream::Advance() (which calls Read() and ++// discards) effectively free for skipped pages. ++// - Peek(): Always falls back to source on cache miss, because PageReader uses ++// Peek() to read Thrift page headers (~30 bytes) which must have real data. ++class CachedInputStream : public ::arrow::io::InputStream { ++ public: ++ CachedInputStream( ++ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache, ++ std::shared_ptr source, ++ int64_t offset, int64_t length) ++ : cache_(std::move(cache)), ++ source_(std::move(source)), ++ base_offset_(offset), ++ length_(length) {} ++ ++ ::arrow::Status Close() override { ++ closed_ = true; ++ return ::arrow::Status::OK(); ++ } ++ ++ bool closed() const override { return closed_; } ++ ++ ::arrow::Result Tell() const override { return position_; } ++ ++ ::arrow::Result Peek(int64_t nbytes) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) { ++ return std::string_view(); ++ } ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ peek_buffer_ = *result; ++ } else { ++ // Peek is used for Thrift page headers (~30 bytes) — must read real data ++ ARROW_ASSIGN_OR_RAISE(peek_buffer_, ++ source_->ReadAt(range.offset, range.length)); ++ } ++ return std::string_view( ++ reinterpret_cast(peek_buffer_->data()), ++ static_cast(peek_buffer_->size())); ++ } ++ ++ ::arrow::Result Read(int64_t nbytes, void* out) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) return 0; ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ auto& buf = *result; ++ memcpy(out, buf->data(), static_cast(buf->size())); ++ position_ += buf->size(); ++ return buf->size(); ++ } ++ // Cache miss: fall back to real I/O from source ++ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length)); ++ memcpy(out, buf->data(), static_cast(buf->size())); ++ position_ += buf->size(); ++ return buf->size(); ++ } ++ ++ ::arrow::Result> Read(int64_t nbytes) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) { ++ return std::make_shared<::arrow::Buffer>(nullptr, 0); ++ } ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ position_ += (*result)->size(); ++ return *result; ++ } ++ // Cache miss: fall back to real I/O from source ++ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length)); ++ position_ += buf->size(); ++ return std::shared_ptr<::arrow::Buffer>(std::move(buf)); ++ } ++ ++ // Override Advance to avoid real I/O for skipped pages. ++ // The default InputStream::Advance() calls Read() and discards the result, ++ // which would trigger source_->ReadAt() on cache miss — defeating page-level ++ // I/O skipping via data_page_filter. Since Advance() is only used to skip ++ // over data that will not be consumed, we can safely just move the position. ++ ::arrow::Status Advance(int64_t nbytes) override { ++ if (nbytes <= 0) { ++ return ::arrow::Status::OK(); ++ } ++ int64_t remaining = length_ - position_; ++ if (remaining <= 0) { ++ return ::arrow::Status::OK(); ++ } ++ position_ += std::min(nbytes, remaining); ++ return ::arrow::Status::OK(); ++ } ++ ++ private: ++ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_; ++ std::shared_ptr source_; ++ int64_t base_offset_; ++ int64_t length_; ++ int64_t position_ = 0; ++ bool closed_ = false; ++ std::shared_ptr<::arrow::Buffer> peek_buffer_; ++}; ++ + // RowGroupReader::Contents implementation for the Parquet file specification + class SerializedRowGroup : public RowGroupReader::Contents { + public: +@@ -242,6 +343,11 @@ + // segments. + PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range)); + stream = std::make_shared<::arrow::io::BufferReader>(buffer); ++ } else if (cached_source_) { ++ // Page-level caching: read through cache with fallback to source. ++ // Advance() is zero-cost for skipped pages via data_page_filter. ++ stream = std::make_shared( ++ cached_source_, source_, col_range.offset, col_range.length); + } else { + stream = properties_.GetStream(source_, col_range.offset, col_range.length); + } +@@ -417,6 +523,26 @@ + return cached_source_->WaitFor(ranges); + } + ++ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options) { ++ cached_source_ = ++ std::make_shared<::arrow::io::internal::ReadRangeCache>(source_, ctx, options); ++ // Do NOT set prebuffered_column_chunks_ bitmap — GetColumnPageReader will ++ // use CachedInputStream path instead of full-chunk BufferReader path. ++ prebuffered_column_chunks_.clear(); ++ PARQUET_THROW_NOT_OK(cached_source_->Cache(ranges)); ++ } ++ ++ ::arrow::Future<> WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const { ++ if (!cached_source_) { ++ return ::arrow::Status::Invalid( ++ "Must call PreBufferRanges before WhenBufferedRanges"); ++ } ++ return cached_source_->WaitFor(ranges); ++ } ++ + // Metadata/footer parsing. Divided up to separate sync/async paths, and to use + // exceptions for error handling (with the async path converting to Future/Status). + +@@ -911,6 +1037,22 @@ + return file->WhenBuffered(row_groups, column_indices); + } + ++void ParquetFileReader::PreBufferRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options) { ++ SerializedFile* file = ++ ::arrow::internal::checked_cast(contents_.get()); ++ file->PreBufferRanges(ranges, ctx, options); ++} ++ ++::arrow::Future<> ParquetFileReader::WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const { ++ SerializedFile* file = ++ ::arrow::internal::checked_cast(contents_.get()); ++ return file->WhenBufferedRanges(ranges); ++} ++ + // ---------------------------------------------------------------------- + // File metadata helpers + diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -211,3 +415,329 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi # and crosscompiling emulator (for try_run() ) if(CMAKE_CROSSCOMPILING_EMULATOR) string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR +@@ -1720,6 +1725,7 @@ macro(build_thrift) + -DWITH_JAVASCRIPT=OFF + -DWITH_LIBEVENT=OFF + -DWITH_NODEJS=OFF ++ -DWITH_OPENSSL=OFF + -DWITH_PYTHON=OFF + -DWITH_QT5=OFF + -DWITH_ZLIB=OFF) +diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake +--- a/cpp/cmake_modules/BuildUtils.cmake ++++ b/cpp/cmake_modules/BuildUtils.cmake +@@ -112,7 +112,7 @@ function(arrow_create_merged_static_lib output_target) + execute_process(COMMAND ${LIBTOOL_MACOS} -V + OUTPUT_VARIABLE LIBTOOL_V_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE) +- if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*") ++ if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools(_ld)?-([0-9.]+).*") + message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}" + ) + endif() + +diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h +--- a/cpp/src/arrow/io/interfaces.h ++++ b/cpp/src/arrow/io/interfaces.h +@@ -210,7 +210,7 @@ + /// \brief Advance or skip stream indicated number of bytes + /// \param[in] nbytes the number to move forward + /// \return Status +- Status Advance(int64_t nbytes); ++ virtual Status Advance(int64_t nbytes); + + /// \brief Return zero-copy string_view to upcoming bytes. + /// +--- a/cpp/src/parquet/arrow/reader.cc ++++ b/cpp/src/parquet/arrow/reader.cc +@@ -254,6 +254,11 @@ + return GetColumn(i, AllRowGroupsFactory(), out); + } + ++ ::arrow::Status GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) override; ++ + Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override { + return FromParquetSchema(reader_->metadata()->schema(), reader_properties_, + reader_->metadata()->key_value_metadata(), out); +@@ -493,10 +498,40 @@ + + ::arrow::Status BuildArray(int64_t length_upper_bound, + std::shared_ptr<::arrow::ChunkedArray>* out) final { ++ if (!out_) { ++ BEGIN_PARQUET_CATCH_EXCEPTIONS ++ RETURN_NOT_OK( ++ TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_)); ++ END_PARQUET_CATCH_EXCEPTIONS ++ } + *out = out_; + return Status::OK(); + } + ++ std::vector LeafColumnIndices() const final { ++ return {input_->column_index()}; ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ if (col_idx != input_->column_index()) return Status::OK(); ++ BEGIN_PARQUET_CATCH_EXCEPTIONS ++ out_ = nullptr; ++ record_reader_->Reset(); ++ record_reader_->Reserve(reserve); ++ return Status::OK(); ++ END_PARQUET_CATCH_EXCEPTIONS ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ if (col_idx != input_->column_index() || num_records <= 0) return 0; ++ return record_reader_->SkipRecords(num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ if (col_idx != input_->column_index() || num_records <= 0) return 0; ++ return record_reader_->ReadRecords(num_records); ++ } ++ + const std::shared_ptr field() override { return field_; } + + private: +@@ -532,6 +567,22 @@ + return storage_reader_->LoadBatch(number_of_records); + } + ++ std::vector LeafColumnIndices() const final { ++ return storage_reader_->LeafColumnIndices(); ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ return storage_reader_->ResetLeaf(col_idx, reserve); ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ return storage_reader_->SkipRecords(col_idx, num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ return storage_reader_->ReadRecords(col_idx, num_records); ++ } ++ + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override { + std::shared_ptr storage; +@@ -576,6 +627,22 @@ + return item_reader_->LoadBatch(number_of_records); + } + ++ std::vector LeafColumnIndices() const final { ++ return item_reader_->LeafColumnIndices(); ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final { ++ return item_reader_->ResetLeaf(col_idx, reserve); ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) final { ++ return item_reader_->SkipRecords(col_idx, num_records); ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) final { ++ return item_reader_->ReadRecords(col_idx, num_records); ++ } ++ + virtual ::arrow::Result> AssembleArray( + std::shared_ptr data) { + if (field_->type()->id() == ::arrow::Type::MAP) { +@@ -709,6 +776,39 @@ + } + return Status::OK(); + } ++ ++ std::vector LeafColumnIndices() const override { ++ std::vector indices; ++ for (const std::unique_ptr& reader : children_) { ++ std::vector child_indices = reader->LeafColumnIndices(); ++ indices.insert(indices.end(), child_indices.begin(), child_indices.end()); ++ } ++ return indices; ++ } ++ ++ ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) override { ++ for (const std::unique_ptr& reader : children_) { ++ RETURN_NOT_OK(reader->ResetLeaf(col_idx, reserve)); ++ } ++ return Status::OK(); ++ } ++ ++ int64_t SkipRecords(int col_idx, int64_t num_records) override { ++ int64_t skipped = 0; ++ for (const std::unique_ptr& reader : children_) { ++ skipped += reader->SkipRecords(col_idx, num_records); ++ } ++ return skipped; ++ } ++ ++ int64_t ReadRecords(int col_idx, int64_t num_records) override { ++ int64_t read = 0; ++ for (const std::unique_ptr& reader : children_) { ++ read += reader->ReadRecords(col_idx, num_records); ++ } ++ return read; ++ } ++ + Status BuildArray(int64_t length_upper_bound, + std::shared_ptr* out) override; + Status GetDefLevels(const int16_t** data, int64_t* length) override; +@@ -1228,6 +1328,23 @@ + std::unique_ptr result; + RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); + *out = std::move(result); ++ return Status::OK(); ++} ++ ++::arrow::Status FileReaderImpl::GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) { ++ RETURN_NOT_OK(BoundsCheckColumn(i)); ++ auto ctx = std::make_shared(); ++ ctx->reader = reader_.get(); ++ ctx->pool = pool_; ++ ctx->iterator_factory = iterator_factory; ++ ctx->filter_leaves = true; ++ ctx->included_leaves = VectorToSharedSet(column_indices); ++ std::unique_ptr result; ++ RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result)); ++ *out = std::move(result); + return Status::OK(); + } + +--- a/cpp/src/parquet/arrow/reader.h ++++ b/cpp/src/parquet/arrow/reader.h +@@ -21,6 +21,7 @@ + // N.B. we don't include async_generator.h as it's relatively heavy + #include + #include ++#include + #include + + #include "parquet/file_reader.h" +@@ -48,9 +49,13 @@ + + class ColumnChunkReader; + class ColumnReader; ++class FileColumnIterator; + struct SchemaManifest; + class RowGroupReader; + ++using FileColumnIteratorFactory = ++ std::function; ++ + /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches. + /// + /// This interfaces caters for different use cases and thus provides different +@@ -136,6 +141,27 @@ + // The indicated column index is relative to the schema + virtual ::arrow::Status GetColumn(int i, std::unique_ptr* out) = 0; + ++ /// \brief Return a ColumnReader with a custom FileColumnIteratorFactory ++ /// and leaf column filtering. ++ /// ++ /// This allows callers to customize page reading behavior (e.g., setting ++ /// data_page_filter for page-level skipping) and to select only specific ++ /// leaf columns within a nested field. The factory is called once per leaf ++ /// column included in column_indices. ++ /// ++ /// \param i top-level field index (same as GetColumn(int i, ...)) ++ /// \param column_indices leaf column indices to include (enables sub-column ++ /// projection within nested types) ++ /// \param iterator_factory factory to create FileColumnIterator per leaf ++ /// \param[out] out the ColumnReader (may be nullptr if all leaves are pruned) ++ virtual ::arrow::Status GetColumn( ++ int i, const std::vector& column_indices, ++ FileColumnIteratorFactory iterator_factory, ++ std::unique_ptr* out) { ++ return ::arrow::Status::NotImplemented( ++ "GetColumn with factory not implemented"); ++ } ++ + /// \brief Return arrow schema for all the columns. + virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0; + +@@ -316,6 +342,43 @@ + // the data available in the file. + virtual ::arrow::Status NextBatch(int64_t batch_size, + std::shared_ptr<::arrow::ChunkedArray>* out) = 0; ++ ++ /// \brief Leaf column indices covered by this (sub)tree, in leaf order. ++ /// ++ /// Used to drive per-leaf row filtering: after page-level skipping each leaf ++ /// lives in its own compressed coordinate space, so callers must reset and ++ /// skip/read each leaf independently rather than in lockstep. ++ virtual std::vector LeafColumnIndices() const { return {}; } ++ ++ /// \brief Reset the leaf identified by col_idx and reserve space for ++ /// `reserve` records (in that leaf's post-page-filter compressed space). ++ /// Must be called before SkipRecords()/ReadRecords() for that leaf, and ++ /// followed by BuildArray() to get the result. ++ virtual ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) { ++ return ::arrow::Status::NotImplemented("ResetLeaf not implemented"); ++ } ++ ++ /// \brief Skip num_records on the leaf identified by col_idx and return the ++ /// number of records actually skipped. Returns 0 when num_records <= 0 or ++ /// col_idx does not belong to this (sub)tree. May throw ParquetException on a ++ /// decode error; callers convert it to Status at the public boundary. ++ virtual int64_t SkipRecords(int col_idx, int64_t num_records) { return 0; } ++ ++ /// \brief Read num_records on the leaf identified by col_idx and return the ++ /// number of records actually read. Values accumulate across successive calls ++ /// until BuildArray() is called. Returns 0 when num_records <= 0 or col_idx ++ /// does not belong to this (sub)tree. May throw ParquetException on a decode ++ /// error; callers convert it to Status at the public boundary. ++ virtual int64_t ReadRecords(int col_idx, int64_t num_records) { return 0; } ++ ++ /// \brief Build the Arrow array from previously loaded data. ++ /// For leaf readers, calls TransferColumnData if not already done. ++ /// For nested readers, assembles the nested array from child arrays. ++ virtual ::arrow::Status BuildArray( ++ int64_t length_upper_bound, ++ std::shared_ptr<::arrow::ChunkedArray>* out) { ++ return ::arrow::Status::NotImplemented("BuildArray not implemented"); ++ } + }; + + /// \brief Experimental helper class for bindings (like Python) that struggle +--- a/cpp/src/parquet/arrow/reader_internal.h ++++ b/cpp/src/parquet/arrow/reader_internal.h +@@ -26,6 +26,7 @@ + #include + #include + ++#include "parquet/arrow/reader.h" + #include "parquet/arrow/schema.h" + #include "parquet/column_reader.h" + #include "parquet/file_reader.h" +@@ -70,7 +71,10 @@ + + virtual ~FileColumnIterator() {} + +- std::unique_ptr<::parquet::PageReader> NextChunk() { ++ /// \brief Fetch the PageReader for the next row group in this iterator's ++ /// range. Virtual so subclasses can decorate the returned PageReader, e.g. ++ /// to install a data_page_filter for I/O-level page skipping. ++ virtual std::unique_ptr<::parquet::PageReader> NextChunk() { + if (row_groups_.empty()) { + return nullptr; + } +@@ -95,9 +99,6 @@ + std::deque row_groups_; + }; + +-using FileColumnIteratorFactory = +- std::function; +- + Status TransferColumnData(::parquet::internal::RecordReader* reader, + const std::shared_ptr<::arrow::Field>& value_field, + const ColumnDescriptor* descr, ::arrow::MemoryPool* pool, diff --git a/cmake_modules/san-config.cmake b/cmake_modules/san-config.cmake index bb078c72..a2ab16ae 100644 --- a/cmake_modules/san-config.cmake +++ b/cmake_modules/san-config.cmake @@ -16,6 +16,11 @@ add_library(paimon_sanitizer_flags INTERFACE) +if(PAIMON_USE_ASAN AND PAIMON_USE_TSAN) + message(FATAL_ERROR "Address Sanitizer and Thread Sanitizer cannot be enabled together" + ) +endif() + if(PAIMON_USE_ASAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=address @@ -27,10 +32,24 @@ if(PAIMON_USE_ASAN) endif() endif() +if(PAIMON_USE_TSAN) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # Bundled dependencies are linked statically into Paimon. Instrument them too so TSAN can + # observe their synchronization primitives and does not report false races at the boundary. + string(APPEND CMAKE_C_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + string(APPEND CMAKE_CXX_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread) + message(STATUS "Thread Sanitizer enabled") + else() + message(WARNING "Thread Sanitizer is only supported for GCC and Clang compilers") + endif() +endif() + if(PAIMON_USE_UBSAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined - -fno-omit-frame-pointer) + target_compile_options(paimon_sanitizer_flags + INTERFACE -fsanitize=undefined -fno-sanitize=vptr + -fno-omit-frame-pointer) target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined) message(STATUS "Undefined Behavior Sanitizer enabled") else() diff --git a/crates/tantivy_ffi/Cargo.lock b/crates/tantivy_ffi/Cargo.lock new file mode 100644 index 00000000..be9056ad --- /dev/null +++ b/crates/tantivy_ffi/Cargo.lock @@ -0,0 +1,1859 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cbindgen" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + +[[package]] +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cedarwood" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" +dependencies = [ + "smallvec", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croaring" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e813b58ac55ac5ccea5ec63beb8c80f37dedd78da3f594c848313415a08c8c" +dependencies = [ + "allocator-api2 0.4.0", + "croaring-sys", +] + +[[package]] +name = "croaring-sys" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f34e9ee8e65c0d46c9d0fe55ce80b477d0bfae4c786c6694687b9c70e8267027" +dependencies = [ + "cc", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "include-flate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969" +dependencies = [ + "include-flate-compress", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff" +dependencies = [ + "libflate", + "zstd", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jieba-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a" +dependencies = [ + "cedarwood", + "fxhash", + "include-flate", + "jieba-macros", + "lazy_static", + "phf", + "regex", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" + +[[package]] +name = "libflate" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "no_std_io2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "paimon_tantivy_ffi" +version = "0.1.0" +dependencies = [ + "cbindgen", + "croaring", + "jieba-rs", + "log", + "tantivy", + "tempfile", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tantivy" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.1", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/tantivy_ffi/Cargo.toml b/crates/tantivy_ffi/Cargo.toml new file mode 100644 index 00000000..ea843bd9 --- /dev/null +++ b/crates/tantivy_ffi/Cargo.toml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "paimon_tantivy_ffi" +version = "0.1.0" +edition = "2021" +description = "C FFI layer wrapping tantivy + jieba-rs for paimon-cpp tantivy-fts global index" +license = "Apache-2.0" +publish = false + +[lib] +name = "paimon_tantivy_ffi" +# staticlib: linked by CMake + Corrosion into libpaimon_tantivy_ffi.a +# rlib: lets `cargo test` use native Rust linkage when building test binaries +crate-type = ["staticlib", "rlib"] + +[dependencies] +tantivy = "0.22" +jieba-rs = "0.7" +croaring = "2.0" +log = "0.4" +tempfile = "3" + +[build-dependencies] +cbindgen = "0.29" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "abort" + +[profile.dev] +# Errors propagate across the FFI via status codes; a Rust panic must abort +# rather than unwind across the FFI boundary. +panic = "abort" diff --git a/crates/tantivy_ffi/build.rs b/crates/tantivy_ffi/build.rs new file mode 100644 index 00000000..72d76b6f --- /dev/null +++ b/crates/tantivy_ffi/build.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! build.rs: runs cbindgen to generate the C header paimon_tantivy_ffi.h. +//! +//! Output path: $OUT_DIR/paimon_tantivy_ffi.h +//! Corrosion (on the CMake side) reads OUT_DIR from cargo metadata and adds the +//! header to the C++ include path. + +use std::env; +use std::path::PathBuf; + +fn main() { + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let header_path = out_dir.join("paimon_tantivy_ffi.h"); + + let cfg = cbindgen::Config::from_file(PathBuf::from(&crate_dir).join("cbindgen.toml")) + .expect("cbindgen.toml must exist at crate root"); + + match cbindgen::Builder::new() + .with_crate(&crate_dir) + .with_config(cfg) + .generate() + { + Ok(bindings) => { + bindings.write_to_file(&header_path); + println!( + "cargo:rerun-if-changed={}", + PathBuf::from(&crate_dir).join("src").display() + ); + println!("cargo:rerun-if-changed=cbindgen.toml"); + // Expose the header directory to Corrosion / the upstream CMake build. + println!("cargo:include={}", out_dir.display()); + eprintln!("cbindgen: wrote {}", header_path.display()); + } + Err(e) => { + // cbindgen failure is not necessarily fatal (e.g. CI skips it when the + // Rust code is unchanged); log a warning and continue. + eprintln!("cbindgen generation failed: {e:?}"); + } + } +} diff --git a/crates/tantivy_ffi/cbindgen.toml b/crates/tantivy_ffi/cbindgen.toml new file mode 100644 index 00000000..c2ab5f13 --- /dev/null +++ b/crates/tantivy_ffi/cbindgen.toml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cbindgen config: Rust FFI -> C header generator. +# Invoked by build.rs, outputs to $OUT_DIR/paimon_tantivy_ffi.h. +# CMake picks up $OUT_DIR via Corrosion and adds it to the C++ include path. + +language = "C" + +# Header banner written at the top of the generated file. +header = """ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * \"License\"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * AUTO-GENERATED by cbindgen from Rust sources under crates/tantivy_ffi - DO NOT EDIT. + * + * C ABI for paimon_tantivy_ffi. + */ +#pragma once +""" + +include_guard = "PAIMON_TANTIVY_FFI_H" +cpp_compat = true +pragma_once = false # already written by hand in the header banner above +documentation = true +documentation_style = "c" +line_length = 100 +tab_width = 4 + +[export] +# No type prefix (Rust type names already carry the PaimonTantivy... prefix). +# Function names already start with paimon_tantivy_ (named that way in the Rust source). +prefix = "" +# Force-export types that only appear as handles / return values; cbindgen does +# not export them by default when no FFI function directly takes/returns them, +# so list them explicitly here. +include = ["PaimonTantivyStatus"] + +[export.rename] +# Rust enum name -> C typedef name (to avoid duplicated prefixes, etc.) + +[fn] +prefix = "" +args = "auto" +rename_args = "None" + +[enum] +rename_variants = "ScreamingSnakeCase" +prefix_with_name = true +derive_helper_methods = false + +[parse] +parse_deps = false diff --git a/crates/tantivy_ffi/rust-toolchain.toml b/crates/tantivy_ffi/rust-toolchain.toml new file mode 100644 index 00000000..4c1d88ae --- /dev/null +++ b/crates/tantivy_ffi/rust-toolchain.toml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pin the Rust toolchain used to build paimon_tantivy_ffi so Corrosion can +# resolve a non-empty toolchain name during a fresh CMake configure. +# +# Only the `channel` is pinned — no extra components, because rustup in +# CI/containers may lack network access to fetch clippy/rustfmt, and build +# doesn't need them. +[toolchain] +channel = "stable" +profile = "minimal" diff --git a/crates/tantivy_ffi/src/buffer.rs b/crates/tantivy_ffi/src/buffer.rs new file mode 100644 index 00000000..2d96e89d --- /dev/null +++ b/crates/tantivy_ffi/src/buffer.rs @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `paimon_tantivy_buffer_t`: Rust-allocated byte buffer returned to C++. +//! +//! Contract: +//! - Buffer is allocated by Rust (as a `Box<[u8]>`) +//! - C++ reads `data[0..len]`, **must not** write past len +//! - C++ must call `paimon_tantivy_buffer_free()` exactly once per non-empty buffer +//! - Empty (len=0) buffer has null `data`; buffer_free accepts it as no-op +//! +//! This struct is #[repr(C)] so cbindgen generates a matching C struct. + +use std::ptr; + +#[repr(C)] +pub struct PaimonTantivyBuffer { + /// Pointer to `len` bytes. Null iff len == 0. + pub data: *mut u8, + /// Number of valid bytes. + pub len: usize, + /// Internal capacity hint for Rust-side reconstruction. C++ treats as opaque. + pub capacity: usize, +} + +impl PaimonTantivyBuffer { + /// Build a buffer from owned bytes; consumes the Vec. + pub(crate) fn from_vec(mut v: Vec) -> Self { + if v.is_empty() { + return Self::empty(); + } + v.shrink_to_fit(); + let len = v.len(); + let capacity = v.capacity(); + let data = v.as_mut_ptr(); + std::mem::forget(v); + Self { data, len, capacity } + } + + pub(crate) fn empty() -> Self { + Self { + data: ptr::null_mut(), + len: 0, + capacity: 0, + } + } +} + +/// Free a buffer returned by any Rust FFI function. Safe to call on an empty +/// buffer (len=0 / data=null). Must only be called once per buffer. +/// +/// SAFETY: `buf` must be either null, or point to a live `paimon_tantivy_buffer_t` +/// produced by this crate and not yet freed. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_buffer_free(buf: *mut PaimonTantivyBuffer) { + if buf.is_null() { + return; + } + let b = unsafe { &mut *buf }; + if b.len != 0 && !b.data.is_null() { + // Reconstruct the Vec and drop it + let v = unsafe { Vec::from_raw_parts(b.data, b.len, b.capacity) }; + drop(v); + } + b.data = ptr::null_mut(); + b.len = 0; + b.capacity = 0; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_has_null_data() { + let b = PaimonTantivyBuffer::empty(); + assert!(b.data.is_null()); + assert_eq!(b.len, 0); + } + + #[test] + fn from_vec_roundtrip() { + let src = vec![1u8, 2, 3, 4, 5]; + let src_clone = src.clone(); + let mut b = PaimonTantivyBuffer::from_vec(src); + assert_eq!(b.len, 5); + assert!(!b.data.is_null()); + let view: &[u8] = unsafe { std::slice::from_raw_parts(b.data, b.len) }; + assert_eq!(view, src_clone.as_slice()); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + assert!(b.data.is_null()); + assert_eq!(b.len, 0); + } + + #[test] + fn free_null_is_noop() { + unsafe { paimon_tantivy_buffer_free(std::ptr::null_mut()) }; + } + + #[test] + fn free_empty_is_noop() { + let mut b = PaimonTantivyBuffer::empty(); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + } + + #[test] + fn stress_alloc_free() { + // LSAN would catch any leak + for i in 0..5_000usize { + let mut b = PaimonTantivyBuffer::from_vec(vec![42u8; i.min(256)]); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + } + } +} diff --git a/crates/tantivy_ffi/src/callback_directory.rs b/crates/tantivy_ffi/src/callback_directory.rs new file mode 100644 index 00000000..12768f32 --- /dev/null +++ b/crates/tantivy_ffi/src/callback_directory.rs @@ -0,0 +1,531 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PaimonCallbackDirectory: streaming tantivy `Directory` backed by C FFI +//! callbacks. Replaces the V1 `PaimonDirectory` (RamDirectory wrapper) with a +//! callback-driven design that mirrors Java paimon-tantivy-jni's `JniDirectory`. +//! +//! ## Why callback-based? +//! +//! Loading the entire archive (100MB+) into `RamDirectory` at reader +//! construction would give ~2x archive peak RAM and pay the whole download +//! cost up front even for small queries. This directory keeps just the +//! `HashMap` layout and issues pread calls through the FFI +//! callback whenever tantivy asks for bytes — peak RAM is ~KB, startup is +//! ~header size. +//! +//! ## Concurrency +//! +//! `read_at` is serialized via `stream_mutex` (same as Java JniDir's +//! `stream_lock`). pread-style callbacks in principle allow concurrent reads, +//! but some `paimon::InputStream` subclasses (notably `JindoInputStream`) +//! have shared-state races, so we play it safe. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::fmt; +use std::io; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError}; +use tantivy::directory::{ + AntiCallToken, Directory, DirectoryLock, FileHandle, Lock, OwnedBytes, TerminatingWrite, + WatchCallback, WatchHandle, WritePtr, +}; +use tantivy::HasLen; + +// ========================================================================= +// FFI types +// ========================================================================= + +/// pread-style callback table passed from C++ at reader construction. +/// +/// `ctx` is an opaque pointer to C++'s `StreamCtx` (holding a +/// `paimon::InputStream`). Rust never dereferences it — only forwards it +/// into the callback functions. `release` is called exactly once when the +/// last `Arc` is dropped. +#[repr(C)] +pub struct PaimonStreamCallbacks { + pub ctx: *mut c_void, + pub read_at: + extern "C" fn(ctx: *mut c_void, offset: u64, len: usize, out_buf: *mut u8) -> i32, + pub release: extern "C" fn(ctx: *mut c_void), +} + +// ========================================================================= +// Internal state +// ========================================================================= + +#[derive(Clone, Debug)] +struct FileMeta { + offset: u64, + length: u64, +} + +/// RAII wrapper owning the FFI callbacks. On drop, invokes `release(ctx)`. +/// Shared across clones of `PaimonCallbackDirectory` via `Arc`. +struct CallbackCtx { + callbacks: PaimonStreamCallbacks, +} + +impl Drop for CallbackCtx { + fn drop(&mut self) { + // Calling an extern "C" fn pointer from safe Rust is legal; the + // contract safety relies on the C++ side providing a valid ctx. + (self.callbacks.release)(self.callbacks.ctx); + } +} + +// Safety: callbacks.ctx is treated as opaque; C++ owner is responsible for +// the ctx being usable across threads. Rust's stream_mutex serializes +// read_at calls, and release is only invoked once (when Arc refcount hits 0). +unsafe impl Send for CallbackCtx {} +unsafe impl Sync for CallbackCtx {} + +// ========================================================================= +// PaimonCallbackDirectory +// ========================================================================= + +#[derive(Clone)] +pub struct PaimonCallbackDirectory { + /// name → (offset, length) in the stream. Immutable after construction. + layout: Arc>, + /// FFI callbacks + their ctx lifetime. + ctx: Arc, + /// tantivy writes small atomic files (`.lock`, in some paths `meta.json`) + /// via `atomic_write`; we keep them in memory instead of pushing back + /// through C++ (read-only archive). Shared across clones. + atomic_data: Arc>>>, + /// Serialize seek+read (mirrors Java JniDir's `stream_lock`) to guard + /// against shared-state races in some InputStream subclasses. + stream_mutex: Arc>, +} + +impl fmt::Debug for PaimonCallbackDirectory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PaimonCallbackDirectory") + .field("files", &self.layout.keys().collect::>()) + .finish() + } +} + +impl PaimonCallbackDirectory { + /// Construct a new directory from the C++-parsed archive layout + callbacks. + /// The ctx ownership transfers to this Directory; `release` is invoked on + /// drop of the last clone. + pub fn new( + entries: Vec<(String, u64, u64)>, + callbacks: PaimonStreamCallbacks, + ) -> Self { + let mut layout = HashMap::with_capacity(entries.len()); + for (name, offset, length) in entries { + layout.insert(PathBuf::from(name), FileMeta { offset, length }); + } + Self { + layout: Arc::new(layout), + ctx: Arc::new(CallbackCtx { callbacks }), + atomic_data: Arc::new(Mutex::new(HashMap::new())), + stream_mutex: Arc::new(Mutex::new(())), + } + } + + /// Perform an FFI pread. Serialized via `stream_mutex`. + fn pread(&self, offset: u64, len: usize) -> io::Result> { + let _guard = self.stream_mutex.lock().map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("stream_mutex poisoned: {e}")) + })?; + let mut buf = vec![0u8; len]; + // Calling extern "C" fn pointer — safe from Rust's POV (ABI is C); + // the contract safety (ctx validity, buffer ownership) is on the C++ side. + let rc = + (self.ctx.callbacks.read_at)(self.ctx.callbacks.ctx, offset, len, buf.as_mut_ptr()); + if rc != 0 { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("pread callback rc={rc} offset={offset} len={len}"), + )); + } + Ok(buf) + } + + /// Sorted file names, for diagnostic / test use. + #[cfg(test)] + pub(crate) fn file_names(&self) -> Vec { + let mut names: Vec = self + .layout + .keys() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } +} + +// ========================================================================= +// FileHandle +// ========================================================================= + +#[derive(Clone)] +struct PaimonCallbackFileHandle { + directory: PaimonCallbackDirectory, + file_offset: u64, + file_length: u64, +} + +impl fmt::Debug for PaimonCallbackFileHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PaimonCallbackFileHandle") + .field("offset", &self.file_offset) + .field("length", &self.file_length) + .finish() + } +} + +impl HasLen for PaimonCallbackFileHandle { + fn len(&self) -> usize { + self.file_length as usize + } +} + +impl FileHandle for PaimonCallbackFileHandle { + fn read_bytes(&self, range: Range) -> io::Result { + let start = self.file_offset + range.start as u64; + let len = range.end - range.start; + let data = self.directory.pread(start, len)?; + Ok(OwnedBytes::new(data)) + } +} + +// ========================================================================= +// Directory trait (13 methods for tantivy 0.22) +// ========================================================================= + +impl Directory for PaimonCallbackDirectory { + fn get_file_handle(&self, path: &Path) -> Result, OpenReadError> { + let meta = self + .layout + .get(path) + .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?; + Ok(Arc::new(PaimonCallbackFileHandle { + directory: self.clone(), + file_offset: meta.offset, + file_length: meta.length, + })) + } + + fn exists(&self, path: &Path) -> Result { + let in_layout = self.layout.contains_key(path); + let in_atomic = self.atomic_data.lock().unwrap().contains_key(path); + Ok(in_layout || in_atomic) + } + + fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { + if let Some(data) = self.atomic_data.lock().unwrap().get(path) { + return Ok(data.clone()); + } + let meta = self + .layout + .get(path) + .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?; + self.pread(meta.offset, meta.length as usize) + .map_err(|e| OpenReadError::wrap_io_error(e, path.to_path_buf())) + } + + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + self.atomic_data + .lock() + .unwrap() + .insert(path.to_path_buf(), data.to_vec()); + Ok(()) + } + + fn delete(&self, _path: &Path) -> Result<(), DeleteError> { + // read-only archive: ignore + Ok(()) + } + + fn open_write(&self, _path: &Path) -> Result { + // tantivy needs this for lock files when opening an index; provide a + // dummy in-memory writer (same trick as Java JniDirectory). + let buf: Vec = Vec::new(); + Ok(io::BufWriter::new(Box::new(VecTerminatingWrite(buf)))) + } + + fn sync_directory(&self) -> io::Result<()> { + Ok(()) + } + + fn acquire_lock(&self, _lock: &Lock) -> Result { + // Read-only: no actual locking. + Ok(DirectoryLock::from(Box::new(()))) + } + + fn watch(&self, _watch_callback: WatchCallback) -> tantivy::Result { + Ok(WatchHandle::empty()) + } +} + +/// Throwaway writer for `open_write` — tantivy creates it for lock files but +/// the bytes never matter in a read-only archive. +struct VecTerminatingWrite(Vec); + +impl io::Write for VecTerminatingWrite { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl TerminatingWrite for VecTerminatingWrite { + fn terminate_ref(&mut self, _token: AntiCallToken) -> io::Result<()> { + Ok(()) + } +} + +// ========================================================================= +// Test support (pub(crate) — used by reader.rs tests too) +// ========================================================================= + +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Mock backend: an in-memory buffer serving pread requests. Counters + /// expose behavior for test assertions (read count / release count). + pub(crate) struct MockBackend { + pub data: Vec, + pub read_count: AtomicUsize, + pub release_count: AtomicUsize, + } + + extern "C" fn mock_read_at( + ctx: *mut c_void, + offset: u64, + len: usize, + out_buf: *mut u8, + ) -> i32 { + let backend = unsafe { &*(ctx as *const MockBackend) }; + backend.read_count.fetch_add(1, Ordering::SeqCst); + let data = &backend.data; + let end = (offset as usize).saturating_add(len); + if end > data.len() { + return 1; // out of range + } + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr().add(offset as usize), out_buf, len); + } + 0 + } + + extern "C" fn mock_release(ctx: *mut c_void) { + // Reclaim the strong ref that `Arc::into_raw` leaked at construction. + let backend = unsafe { Arc::from_raw(ctx as *const MockBackend) }; + backend.release_count.fetch_add(1, Ordering::SeqCst); + // `arc` drops here → decrement; test still holds its own clone. + } + + /// Build a mock-backed directory for tests. Returns (dir, backend clone). + /// The backend Arc is shared — drop the directory to trigger release. + pub(crate) fn build_mock_directory( + data: Vec, + entries: Vec<(String, u64, u64)>, + ) -> (PaimonCallbackDirectory, Arc) { + let backend = Arc::new(MockBackend { + data, + read_count: AtomicUsize::new(0), + release_count: AtomicUsize::new(0), + }); + let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void; + let cb = PaimonStreamCallbacks { + ctx: ctx_ptr, + read_at: mock_read_at, + release: mock_release, + }; + let dir = PaimonCallbackDirectory::new(entries, cb); + (dir, backend) + } + + /// Build mock callbacks (+ a backend clone) without wrapping them in a + /// directory, for tests that drive the FFI entry points directly. + pub(crate) fn make_mock_callbacks(data: Vec) -> (PaimonStreamCallbacks, Arc) { + let backend = Arc::new(MockBackend { + data, + read_count: AtomicUsize::new(0), + release_count: AtomicUsize::new(0), + }); + let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void; + let cb = PaimonStreamCallbacks { + ctx: ctx_ptr, + read_at: mock_read_at, + release: mock_release, + }; + (cb, backend) + } + + /// Parse a packed archive blob (BE, no version header, matching + /// `writer::pack_index_dir`) and build a mock-backed directory. Used by + /// `reader.rs::tests` since writer.finish currently still returns a Vec. + pub(crate) fn build_directory_from_archive( + packed: Vec, + ) -> (PaimonCallbackDirectory, Arc) { + let entries = parse_archive_header(&packed); + build_mock_directory(packed, entries) + } + + /// Parse the archive header — mirrors the layout that + /// C++ `ArchiveLayout::Parse` produces in production. + fn parse_archive_header(bytes: &[u8]) -> Vec<(String, u64, u64)> { + let mut off = 0usize; + let file_count = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let mut entries = Vec::with_capacity(file_count); + for _ in 0..file_count { + let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let name = + std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned(); + off += nlen; + let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as u64; + off += 8; + let data_offset = off as u64; + entries.push((name, data_offset, flen)); + off += flen as usize; + } + entries + } +} + +#[cfg(test)] +mod tests { + use super::test_support::*; + use super::*; + + #[test] + fn file_handle_reads_correct_bytes() { + let data = b"hello world".to_vec(); + let entries = vec![("foo.txt".to_string(), 0, 11)]; + let (dir, _backend) = build_mock_directory(data, entries); + + let handle = dir.get_file_handle(Path::new("foo.txt")).unwrap(); + let bytes = handle.read_bytes(0..5).unwrap(); + assert_eq!(&bytes[..], b"hello"); + let bytes = handle.read_bytes(6..11).unwrap(); + assert_eq!(&bytes[..], b"world"); + } + + #[test] + fn missing_file_returns_error() { + let (dir, _backend) = build_mock_directory(vec![], vec![]); + let err = dir.get_file_handle(Path::new("nonexistent")).unwrap_err(); + match err { + OpenReadError::FileDoesNotExist(p) => { + assert_eq!(p.to_string_lossy(), "nonexistent") + } + other => panic!("expected FileDoesNotExist, got {other:?}"), + } + } + + #[test] + fn pread_out_of_range_propagates_error() { + let data = b"short".to_vec(); + let entries = vec![("bad.txt".to_string(), 0, 100)]; // length exceeds data + let (dir, _backend) = build_mock_directory(data, entries); + let handle = dir.get_file_handle(Path::new("bad.txt")).unwrap(); + let err = handle.read_bytes(0..100).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::Other); + } + + #[test] + fn atomic_write_read_roundtrip_and_exists() { + let (dir, _backend) = build_mock_directory(vec![], vec![]); + dir.atomic_write(Path::new(".lock"), b"locked").unwrap(); + let data = dir.atomic_read(Path::new(".lock")).unwrap(); + assert_eq!(data, b"locked"); + assert!(dir.exists(Path::new(".lock")).unwrap()); + assert!(!dir.exists(Path::new("gone")).unwrap()); + } + + #[test] + fn release_called_exactly_once_on_last_drop() { + let entries = vec![("a".to_string(), 0, 5)]; + let (dir, backend) = build_mock_directory(b"hello".to_vec(), entries); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0); + drop(dir); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn cloned_directory_shares_ctx_and_atomic_data() { + let (dir, backend) = build_mock_directory(vec![], vec![]); + let dir2 = dir.clone(); + dir.atomic_write(Path::new("x"), b"hello").unwrap(); + assert!(dir2.exists(Path::new("x")).unwrap()); // shared atomic_data + drop(dir); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0); // ctx still held by dir2 + drop(dir2); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn concurrent_pread_results_correct_under_stream_mutex() { + use std::thread; + + let data: Vec = (0..1000).map(|i| (i % 256) as u8).collect(); + let entries = vec![("data".to_string(), 0, 1000)]; + let (dir, backend) = build_mock_directory(data.clone(), entries); + let handle: Arc = + dir.get_file_handle(Path::new("data")).unwrap(); + + let threads: Vec<_> = (0..8) + .map(|_| { + let h = handle.clone(); + let expected = data.clone(); + thread::spawn(move || { + for _ in 0..20 { + let bytes = h.read_bytes(100..200).unwrap(); + assert_eq!(&bytes[..], &expected[100..200]); + } + }) + }) + .collect(); + + for t in threads { + t.join().unwrap(); + } + assert_eq!( + backend.read_count.load(std::sync::atomic::Ordering::SeqCst), + 8 * 20 + ); + } + + #[test] + fn file_names_sorted() { + let entries = vec![ + ("z.idx".to_string(), 0, 10), + ("a.meta".to_string(), 10, 20), + ("m.term".to_string(), 30, 5), + ]; + let (dir, _backend) = build_mock_directory(vec![0u8; 100], entries); + let names = dir.file_names(); + assert_eq!(names, vec!["a.meta", "m.term", "z.idx"]); + } +} diff --git a/crates/tantivy_ffi/src/error.rs b/crates/tantivy_ffi/src/error.rs new file mode 100644 index 00000000..36c36f48 --- /dev/null +++ b/crates/tantivy_ffi/src/error.rs @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Error model for paimon_tantivy_ffi. +//! +//! Contract: +//! - Every fallible FFI function returns `paimon_tantivy_status_t` +//! - Failure sets `last_error` (thread-local) with human-readable text +//! - C++ calls `paimon_tantivy_last_error()` after a non-OK status to fetch text +//! - Pointer returned by `last_error()` is thread-local and valid until the +//! next failing FFI call on the same thread. C++ must NOT free it. + +use std::cell::RefCell; +use std::ffi::c_char; +use std::ffi::CString; + +/// Status codes. Values are stable ABI; append-only. +#[repr(i32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum PaimonTantivyStatus { + Ok = 0, + InvalidArgument = 1, + NotFound = 2, + IoError = 3, + Unsupported = 4, + TokenizerError = 5, + QueryParseError = 6, + IndexFormatError = 7, + InternalError = 99, +} + +thread_local! { + /// Pre-allocated empty string so `paimon_tantivy_last_error()` can always + /// return a valid non-null pointer. + static LAST_ERROR: RefCell = RefCell::new(CString::new("").unwrap()); +} + +/// Record an error message for the current thread. Called by fallible FFI +/// functions right before returning a non-OK status. +pub(crate) fn set_last_error(msg: impl Into) { + // Interior nul bytes would make CString::new fail; strip them as a safety net. + let s: String = msg.into().replace('\0', "\u{FFFD}"); + LAST_ERROR.with(|cell| { + // CString::new clones the bytes and appends a nul terminator. + *cell.borrow_mut() = CString::new(s).unwrap_or_else(|_| CString::new("").unwrap()); + }); +} + +/// Clear the current thread's error slot. Called at the top of fallible APIs +/// so a subsequent successful call doesn't return stale text. +#[allow(dead_code)] +pub(crate) fn clear_last_error() { + LAST_ERROR.with(|cell| { + *cell.borrow_mut() = CString::new("").unwrap(); + }); +} + +/// Macro that wraps a `Result`-returning block: sets last_error on +/// Err and returns the given status code; returns Ok value on success. +#[macro_export] +macro_rules! ffi_try { + ($expr:expr, $err_status:expr) => {{ + match $expr { + Ok(v) => v, + Err(e) => { + $crate::error::set_last_error(format!("{e}")); + return $err_status; + } + } + }}; +} + +/// Return the last error text for the calling thread. Always non-null; returns +/// pointer to "" when there is no error recorded yet. Pointer is thread-local; +/// C++ must NOT free it; treat as valid until the next failing FFI call on +/// the same thread. +#[no_mangle] +pub extern "C" fn paimon_tantivy_last_error() -> *const c_char { + LAST_ERROR.with(|cell| cell.borrow().as_ptr()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn initial_last_error_is_empty() { + let ptr = paimon_tantivy_last_error(); + assert!(!ptr.is_null()); + let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap(); + assert_eq!(s, ""); + } + + #[test] + fn set_then_retrieve() { + set_last_error("boom"); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "boom"); + } + + #[test] + fn clear_resets_to_empty() { + set_last_error("x"); + clear_last_error(); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, ""); + } + + #[test] + fn embedded_nul_is_stripped() { + set_last_error("a\0b"); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "a\u{FFFD}b"); + } + + #[test] + fn thread_local_isolation() { + set_last_error("main"); + let t = std::thread::spawn(|| { + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + s.to_owned() + }) + .join() + .unwrap(); + assert_eq!(t, ""); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "main"); + } +} diff --git a/crates/tantivy_ffi/src/handle.rs b/crates/tantivy_ffi/src/handle.rs new file mode 100644 index 00000000..7bd98e73 --- /dev/null +++ b/crates/tantivy_ffi/src/handle.rs @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Opaque handle helpers. +//! +//! Contract: +//! - Rust creates handles with `Box::into_raw(Box::new(T))` +//! - C++ must free with the matching `xxx_free(*mut T)` function, once +//! - Functions accepting handles treat null as invalid argument + +use std::ffi::c_void; + +/// Consume `T`, return a raw opaque pointer suitable for C++. +#[inline] +pub(crate) fn into_handle(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} + +/// Reconstitute a `Box` from an FFI-provided pointer and drop it. +/// SAFETY: caller must pass a pointer previously returned by `into_handle::`, +/// and must not use it again after this call. +#[inline] +pub(crate) unsafe fn free_handle(handle: *mut T) { + if handle.is_null() { + return; + } + drop(unsafe { Box::from_raw(handle) }); +} + +/// Borrow an `&T` from an FFI-provided pointer. Returns None on null. +/// SAFETY: caller must ensure the pointer was previously returned by +/// `into_handle::` and is still alive (not freed). +#[inline] +pub(crate) unsafe fn borrow_handle<'a, T>(handle: *const T) -> Option<&'a T> { + if handle.is_null() { + None + } else { + Some(unsafe { &*handle }) + } +} + +/// Borrow `&mut T` from an FFI-provided pointer. Returns None on null. +/// SAFETY: same as `borrow_handle`, plus caller must ensure there is no +/// concurrent access via another pointer (writer/reader handles are +/// documented as thread-unsafe). +#[inline] +pub(crate) unsafe fn borrow_handle_mut<'a, T>(handle: *mut T) -> Option<&'a mut T> { + if handle.is_null() { + None + } else { + Some(unsafe { &mut *handle }) + } +} + +/// Opaque ctx pointer from C++ (passed through to Rust Directory callbacks). +/// Type-erased on purpose: only C++ side knows the concrete type. +pub(crate) type OpaqueCtx = *mut c_void; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn into_then_free() { + struct X(i32); + let h: *mut X = into_handle(X(42)); + assert!(!h.is_null()); + unsafe { free_handle(h) }; + // no leak (LSAN would catch if compiled with sanitizers) + } + + #[test] + fn free_null_is_noop() { + let h: *mut i32 = std::ptr::null_mut(); + unsafe { free_handle(h) }; + } + + #[test] + fn borrow_roundtrip() { + let h = into_handle(42i32); + unsafe { + assert_eq!(*borrow_handle(h as *const i32).unwrap(), 42); + *borrow_handle_mut(h).unwrap() = 7; + assert_eq!(*borrow_handle(h as *const i32).unwrap(), 7); + free_handle(h); + } + } + + #[test] + fn borrow_null_is_none() { + unsafe { + assert!(borrow_handle::(std::ptr::null()).is_none()); + assert!(borrow_handle_mut::(std::ptr::null_mut()).is_none()); + } + } + + #[test] + fn stress_many_create_destroy() { + // smoke stress: many allocations, no leak + for i in 0..10_000 { + let h = into_handle(vec![i; 8]); + unsafe { + let v = borrow_handle(h as *const Vec).unwrap(); + assert_eq!(v.len(), 8); + free_handle(h); + } + } + } +} diff --git a/crates/tantivy_ffi/src/lib.rs b/crates/tantivy_ffi/src/lib.rs new file mode 100644 index 00000000..df4b80a5 --- /dev/null +++ b/crates/tantivy_ffi/src/lib.rs @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! paimon_tantivy_ffi: C ABI layer for tantivy + jieba-rs, +//! consumed by paimon-cpp's `tantivy-fulltext` global index. +//! +//! Modules: error / handle / buffer / log (common FFI layer), tokenizer, +//! writer, callback directory, and reader (query). + +#![deny(unsafe_op_in_unsafe_fn)] + +use std::ffi::c_char; + +pub mod error; +pub mod handle; +pub mod buffer; +pub mod log_bridge; +pub mod tokenizer; +pub mod writer; +pub mod callback_directory; +pub mod reader; + +// Re-export public FFI symbols at crate root so cbindgen picks them up. +pub use buffer::{paimon_tantivy_buffer_free, PaimonTantivyBuffer}; +pub use error::{paimon_tantivy_last_error, PaimonTantivyStatus}; +pub use log_bridge::{ + paimon_tantivy_clear_log_callback, paimon_tantivy_set_log_callback, PaimonTantivyLogFn, +}; +pub use tokenizer::{ + paimon_tantivy_tokenizer_free, paimon_tantivy_tokenizer_new, + paimon_tantivy_tokenizer_tokenize, PaimonJiebaTokenizer, +}; +pub use writer::{ + paimon_tantivy_writer_add, paimon_tantivy_writer_finish_streaming, + paimon_tantivy_writer_free, paimon_tantivy_writer_new, PaimonTantivyWriter, + PaimonWriteCallbacks, +}; +pub use callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks}; +pub use reader::{ + paimon_tantivy_reader_free, paimon_tantivy_reader_new_streaming, + paimon_tantivy_reader_search, PaimonTantivyReader, +}; + +/// Semantic version of this crate, **'static lifetime**; C++ must NOT free. +/// Format: `""` (git sha postfix can be added later via build.rs). +/// Returned as a NUL-terminated UTF-8 C string. +#[no_mangle] +pub extern "C" fn paimon_tantivy_version() -> *const c_char { + concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn version_is_non_empty() { + let ptr = paimon_tantivy_version(); + assert!(!ptr.is_null()); + let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap(); + assert!(!s.is_empty(), "version must be non-empty"); + assert!(s.contains('.'), "version must look like semver, got {s:?}"); + } + + #[test] + fn tantivy_and_jieba_are_linked() { + let _ = tantivy::schema::Schema::builder(); + let _ = jieba_rs::Jieba::new(); + } + + #[test] + fn croaring_serialize_roundtrip() { + use croaring::Bitmap; + let mut b = Bitmap::new(); + b.add(42); + b.add(100); + let bytes = b.serialize::(); + let b2 = Bitmap::deserialize::(&bytes); + assert_eq!(b.cardinality(), b2.cardinality()); + } +} diff --git a/crates/tantivy_ffi/src/log_bridge.rs b/crates/tantivy_ffi/src/log_bridge.rs new file mode 100644 index 00000000..6b6b286f --- /dev/null +++ b/crates/tantivy_ffi/src/log_bridge.rs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Log bridge: tantivy internally emits log records via the `log` crate +//! (via `tantivy::debug` / `info` etc.). This module registers a global +//! `log::Log` implementation that forwards records to a C callback. +//! +//! Contract: +//! - C++ calls `paimon_tantivy_set_log_callback(cb)` once at process startup +//! - Passing null unregisters (reverts to stderr) +//! - Callback receives (level, msg_ptr, msg_len); pointer is non-null, +//! UTF-8, NOT null-terminated, valid only for the duration of the call +//! - Level mapping: 0=trace 1=debug 2=info 3=warn 4=error +//! - Callback must be thread-safe: tantivy writes from worker threads +//! +//! NOTE: tantivy uses `tracing` in newer versions and `log` in others. +//! Our current `tantivy = "0.22"` uses `log`. +//! If a future upgrade switches to `tracing`, install a `tracing-log` +//! bridge here. + +use std::ffi::c_char; +use std::sync::atomic::{AtomicPtr, Ordering}; + +pub type PaimonTantivyLogFn = extern "C" fn(level: i32, msg: *const c_char, len: usize); + +static CALLBACK: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); + +struct LogBridge; + +impl log::Log for LogBridge { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + let level = match record.level() { + log::Level::Trace => 0, + log::Level::Debug => 1, + log::Level::Info => 2, + log::Level::Warn => 3, + log::Level::Error => 4, + }; + let msg = format!("[{}] {}", record.target(), record.args()); + let ptr = CALLBACK.load(Ordering::Acquire); + if ptr.is_null() { + // Fallback: stderr + eprintln!("{msg}"); + return; + } + // SAFETY: ptr was installed as PaimonTantivyLogFn via transmute below + let cb: PaimonTantivyLogFn = unsafe { std::mem::transmute(ptr) }; + cb(level, msg.as_ptr() as *const c_char, msg.len()); + } + + fn flush(&self) {} +} + +static LOGGER: LogBridge = LogBridge; + +/// Install a non-null callback. First call also registers `LogBridge` as +/// the global `log` crate sink. Subsequent calls swap the callback atomically. +/// Thread-safety: safe to call from any thread. +/// +/// Note: we use separate `set`/`clear` functions instead of `Option` +/// because cbindgen translates `Option` into an opaque struct +/// rather than a nullable C function pointer. +#[no_mangle] +pub extern "C" fn paimon_tantivy_set_log_callback(cb: PaimonTantivyLogFn) { + let ptr = cb as *mut (); + CALLBACK.store(ptr, Ordering::Release); + let _ = log::set_logger(&LOGGER); + log::set_max_level(log::LevelFilter::Info); +} + +/// Clear the installed callback (revert to Rust-side stderr fallback). +#[no_mangle] +pub extern "C" fn paimon_tantivy_clear_log_callback() { + CALLBACK.store(std::ptr::null_mut(), Ordering::Release); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + // Simple test callback that counts invocations + static COUNT: AtomicUsize = AtomicUsize::new(0); + extern "C" fn counting_cb(_: i32, _: *const c_char, _: usize) { + COUNT.fetch_add(1, Ordering::SeqCst); + } + + #[test] + fn install_then_log() { + COUNT.store(0, Ordering::SeqCst); + paimon_tantivy_set_log_callback(counting_cb); + log::info!("hello"); + assert!(COUNT.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn clear_reverts_to_stderr() { + paimon_tantivy_set_log_callback(counting_cb); + paimon_tantivy_clear_log_callback(); + log::warn!("goes to stderr"); + } +} diff --git a/crates/tantivy_ffi/src/reader.rs b/crates/tantivy_ffi/src/reader.rs new file mode 100644 index 00000000..033b7f25 --- /dev/null +++ b/crates/tantivy_ffi/src/reader.rs @@ -0,0 +1,1398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PaimonTantivyReader: query side of tantivy-fulltext. +//! +//! Constructs a tantivy Index from a packed-blob produced by writer.rs (via +//! PaimonDirectory), registers the same `paimon_jieba` tokenizer, and runs +//! one of 5 search types (mirrors `paimon::FullTextSearch::SearchType`): +//! +//! 1 MATCH_ALL — tokenize query, BooleanQuery (Must) +//! 2 MATCH_ANY — tokenize query, BooleanQuery (Should) +//! 3 PHRASE — tokenize query, PhraseQuery +//! 4 PREFIX — original + case-normalized RegexQuery `.*` (no tokenization) +//! 5 WILDCARD — original + case-normalized RegexQuery from glob pattern +//! (`*` → `.*`, `?` → `.`, others escaped) +//! +//! For paimon-java compatibility, row_id is stored as an explicit u64 field +//! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id +//! via `fast_fields().u64("row_id").first(doc_id)` per segment. +//! +//! FFI return format (little-endian, **doc identifiers are u64 row_ids**): +//! `[u8 has_scores | u64 count | u64 row_id[count] | optional f32 score[count]]` + +use std::ffi::{c_char, CStr}; +use std::path::Path; + +use croaring::{Portable, Treemap}; +use tantivy::collector::{Collector, SegmentCollector}; +use tantivy::columnar::Column; +use tantivy::query::{ + BooleanQuery, DisjunctionMaxQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery, +}; +use tantivy::schema::{Field, IndexRecordOption}; +use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal, + SegmentReader, Term}; + +use crate::buffer::PaimonTantivyBuffer; +use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks}; +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle_mut, free_handle, into_handle}; +use crate::tokenizer::{normalize_case, PaimonJiebaTokenizer, TokenizeMode}; +use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME}; + +/// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync +/// with include/paimon/predicate/full_text_search.h. +#[repr(i32)] +#[derive(Clone, Copy, Debug)] +pub enum SearchType { + MatchAll = 1, + MatchAny = 2, + Phrase = 3, + Prefix = 4, + Wildcard = 5, +} + +impl SearchType { + fn from_i32(v: i32) -> Option { + match v { + 1 => Some(Self::MatchAll), + 2 => Some(Self::MatchAny), + 3 => Some(Self::Phrase), + 4 => Some(Self::Prefix), + 5 => Some(Self::Wildcard), + _ => None, + } + } +} + +pub struct PaimonTantivyReader { + /// Held alive so `IndexReader::searcher()` + `index.tokenizers()` stay + /// usable for the reader's lifetime. + index: Index, + reader: IndexReader, + text_field: Field, + /// Name of the tokenizer the `text` field is actually bound to in the open + /// index's schema (read from `meta.json` at construction time). Query-side + /// tokenization looks this up in `index.tokenizers()` every time + tokenizer_name: String, +} + +impl PaimonTantivyReader { + /// Construct a reader from a pre-built callback-backed Directory. + /// Layout (file names + offsets + lengths) must come from the caller + /// (C++ side `ArchiveLayout::Parse`); Rust does not re-parse the archive. + pub fn new( + directory: PaimonCallbackDirectory, + mode: TokenizeMode, + with_position: bool, + dict_dir: &Path, + ) -> Result { + let index = Index::open(directory) + .map_err(|e| format!("tantivy::Index::open: {e}"))?; + + // Resolve fields by their fixed names (schema is `row_id` + `text`). + let schema = index.schema(); + let text_field = schema.get_field(PAIMON_TEXT_FIELD_NAME).map_err(|e| { + format!("tantivy index missing '{PAIMON_TEXT_FIELD_NAME}' field: {e}") + })?; + + // Read the tokenizer name the `text` field was actually written with + // (lives in meta.json's schema). Auto-aligns cpp query-side tokenizer + // with whatever the writer side used. + let tokenizer_name = match schema.get_field_entry(text_field).field_type() { + tantivy::schema::FieldType::Str(text_options) => text_options + .get_indexing_options() + .map(|io| io.tokenizer().to_string()) + .unwrap_or_else(|| "default".to_string()), + other => { + return Err(format!( + "text field has non-TEXT type: {other:?} (schema corrupted?)" + )); + } + }; + + // Only register paimon_jieba if the index actually uses it. The + // tantivy-builtin "default" / "raw" / "en_stem" etc. are pre-registered + // by the TokenizerManager — no setup needed for those. + if tokenizer_name == PAIMON_TOKENIZER_NAME { + // `Path::is_empty` is unstable; check via OsStr. + if dict_dir.as_os_str().is_empty() { + return Err(format!( + "paimon_jieba tokenizer required by archive schema but dict dir \ + is empty — set the PAIMON_JIEBA_DICT_DIR env var to a directory \ + containing jieba.dict.utf8 / hmm_model.utf8 / user.dict.utf8 / \ + idf.utf8 / stop_words.utf8" + )); + } + let jieba = PaimonJiebaTokenizer::new(dict_dir, mode, with_position) + .map_err(|e| format!("create paimon_jieba tokenizer: {e}"))?; + index.tokenizers().register(PAIMON_TOKENIZER_NAME, jieba); + } else { + // For other known-safe names we trust tantivy's builtin registry. + // `mode` / `dict_dir` are unused in this branch — no-op; we still + // require them in the ABI for backward-compat with the jieba case. + let _ = (mode, dict_dir); + } + + // Sanity: the tokenizer MUST be resolvable now; otherwise query-time + // lookup fails mid-flight. + if index.tokenizers().get(&tokenizer_name).is_none() { + return Err(format!( + "tokenizer {tokenizer_name:?} referenced by text field is not \ + registered; add it to TokenizerManager before opening the reader" + )); + } + + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into() + .map_err(|e| format!("build IndexReader: {e}"))?; + + Ok(Self { + index, + reader, + text_field, + tokenizer_name, + }) + } + + /// Tokenize the query string using the *same* tokenizer the index's text + /// field was built with. Looks up `self.tokenizer_name` in the index's + /// `TokenizerManager` — which was populated by `new()` with either + /// `paimon_jieba` (if cpp wrote the index) or a tantivy builtin like + /// `default` (if paimon-java wrote it). + fn tokenize_query(&self, query: &str) -> Vec { + // `TokenizerManager::get` returns a fresh clone per call — safe to use + // across threads / calls. If the tokenizer was missing we'd have + // failed in `new()`; we still defend with `unwrap_or_default`. + let mut analyzer = match self.index.tokenizers().get(&self.tokenizer_name) { + Some(a) => a, + None => return Vec::new(), + }; + let mut stream = analyzer.token_stream(query); + let mut out = Vec::new(); + while stream.advance() { + out.push(stream.token().text.clone()); + } + out + } + + fn build_match_query(&self, query: &str, occur: Occur) -> Result, String> { + let terms = self.tokenize_query(query); + if terms.is_empty() { + return Err(format!("query {query:?} produced no tokens after analysis")); + } + if terms.len() == 1 { + let term = Term::from_field_text(self.text_field, &terms[0]); + return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs))); + } + let clauses: Vec<(Occur, Box)> = terms + .iter() + .map(|t| { + let term = Term::from_field_text(self.text_field, t); + let q: Box = + Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs)); + (occur, q) + }) + .collect(); + Ok(Box::new(BooleanQuery::new(clauses))) + } + + fn build_phrase_query(&self, query: &str) -> Result, String> { + let terms = self.tokenize_query(query); + if terms.is_empty() { + return Err(format!("phrase query {query:?} produced no tokens")); + } + if terms.len() == 1 { + // PhraseQuery requires >=2 terms in tantivy; degrade to TermQuery. + let term = Term::from_field_text(self.text_field, &terms[0]); + return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqsAndPositions))); + } + let tantivy_terms: Vec = terms + .iter() + .map(|t| Term::from_field_text(self.text_field, t)) + .collect(); + Ok(Box::new(PhraseQuery::new(tantivy_terms))) + } + + fn build_prefix_query(&self, query: &str) -> Result, String> { + if query.is_empty() { + return Err("prefix query is empty".into()); + } + // Mirror lucene-fts: don't tokenize the prefix. Retain the original form for mixed + // ASCII/CJK indexed terms, and add the normalized form for pure ASCII terms. + let normalized_query = normalize_case(query); + let create_query = |value: &str| -> Result, String> { + let pattern = format!("{}.*", regex_escape(value)); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from prefix {value:?}: {e}")) + }; + let original = create_query(query)?; + if normalized_query == query { + return Ok(original); + } + let normalized = create_query(&normalized_query)?; + Ok(Box::new(DisjunctionMaxQuery::new(vec![ + original, normalized, + ]))) + } + + fn build_wildcard_query(&self, query: &str) -> Result, String> { + if query.is_empty() { + return Err("wildcard query is empty".into()); + } + let normalized_query = normalize_wildcard_query(query); + let create_query = |value: &str| -> Result, String> { + let pattern = wildcard_to_regex(value); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from wildcard {value:?} (pattern {pattern}): {e}")) + }; + let original = create_query(query)?; + if normalized_query == query { + return Ok(original); + } + let normalized = create_query(&normalized_query)?; + Ok(Box::new(DisjunctionMaxQuery::new(vec![ + original, normalized, + ]))) + } + + fn build_query(&self, search_type: SearchType, query: &str) -> Result, String> { + match search_type { + SearchType::MatchAll => self.build_match_query(query, Occur::Must), + SearchType::MatchAny => self.build_match_query(query, Occur::Should), + SearchType::Phrase => self.build_phrase_query(query), + SearchType::Prefix => self.build_prefix_query(query), + SearchType::Wildcard => self.build_wildcard_query(query), + } + } + + /// Return all matching row_ids (no scoring, no limit, no pre_filter). + /// row_ids come from the explicit `row_id` u64 fast field, supporting + /// multi-segment indexes (e.g. produced by paimon-java without force-merge). + pub fn search_all(&self, search_type: SearchType, query: &str) -> Result, String> { + let q = self.build_query(search_type, query)?; + let searcher = self.reader.searcher(); + let mut ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + /// Four-path dispatch on `(with_score, limit)`. + /// + /// | with_score | limit | path | collector | sort | truncate | output score | + /// |------------|--------|------|------------------------|----------------|----------|--------------| + /// | false | None | A | RowIdCollector | row_id asc | — | ❌ | + /// | false | Some(n)| B | AllScoredCollector | score desc | top n | ❌ (dropped) | + /// | true | None | C | AllScoredCollector | row_id asc | — | ✅ | + /// | true | Some(n)| D | AllScoredCollector | score desc | top n | ✅ | + /// + /// Pre-filter is a `Treemap` of paimon row_ids (not tantivy doc_ids), applied BEFORE + /// truncation so high-score matches outside the filter don't crowd out valid ones. + /// + /// **v0.2 contract change**: previously `limit.is_some()` implicitly triggered scoring; now + /// scoring is gated solely by `with_score`. + pub fn search_with_limit_and_filter( + &self, + search_type: SearchType, + query: &str, + with_score: bool, + limit: Option, + pre_filter: Option<&Treemap>, + min_score: Option, + ) -> Result)>, String> { + let q = self.build_query(search_type, query)?; + let searcher = self.reader.searcher(); + match (with_score, limit) { + // Path A: all rows, no score. RowIdCollector reads the `row_id` fast + // field inline per segment (opened once), avoiding a DocSetCollector + // HashSet and per-doc handle — hot path for high-cardinality counts. + (false, None) => { + let mut row_ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + if let Some(filter) = pre_filter { + row_ids.retain(|id| filter.contains(*id)); + } + row_ids.sort_unstable(); + row_ids.dedup(); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } + // Path B: any N matches, unscored. Used by SR's `WHERE MATCH ... LIMIT N` (no + // ORDER BY): pushes the limit down so each shard stops collecting once N hits + // are gathered per segment instead of materialising the full posting list. + // If the caller wants top-N by BM25 they should set `with_score=true` (Path D) + // and ignore the score values. + (false, Some(n)) => { + if n == 0 { + return Ok(Vec::new()); + } + if min_score.is_some() { + // min_score requires scoring — fall back to collect_scored path + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + let truncated = Self::sort_by_score_desc_truncate(filtered, n); + Ok(truncated.into_iter().map(|(_, id)| (id, None)).collect()) + } else if let Some(filter) = pre_filter { + // pre_filter present: it MUST be applied to the full match set + // before truncation. LimitedDocSetCollector stops after the + // first N raw matches, which could all be filtered out while + // valid matches exist further down the posting list — that + // would under-return (fewer than N, or even empty). So collect + // every matching row_id (filter-aware), then truncate to N. + let mut row_ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + row_ids.retain(|id| filter.contains(*id)); + row_ids.sort_unstable(); + row_ids.dedup(); + row_ids.truncate(n); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } else { + // No pre_filter: fast path — stop collecting once N matches are + // gathered per segment instead of materialising the full posting list. + let collector = LimitedDocSetCollector::new(n); + let mut docset = searcher + .search(&*q, &collector) + .map_err(|e| format!("tantivy search: {e}"))?; + let mut by_segment: std::collections::HashMap> = + std::collections::HashMap::new(); + for addr in docset.drain(..) { + by_segment.entry(addr.segment_ord).or_default().push(addr.doc_id); + } + let mut row_ids: Vec = Vec::new(); + for (segment_ord, doc_ids) in by_segment.iter() { + let segment_reader = searcher.segment_reader(*segment_ord); + let fast = segment_reader + .fast_fields() + .u64(PAIMON_ROW_ID_FIELD_NAME) + .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}", + segment_ord))?; + for &doc_id in doc_ids { + row_ids.push(fast.first(doc_id).unwrap_or(0)); + } + } + row_ids.sort_unstable(); + row_ids.dedup(); + row_ids.truncate(n); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } + } + // Path C: all rows + all scores, sorted by row_id asc to match the + // BitmapScoredGlobalIndexResult contract (bitmap iter order == score order). + (true, None) => { + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + filtered.sort_unstable_by(|a, b| a.1.cmp(&b.1)); + Ok(filtered.into_iter().map(|(s, id)| (id, Some(s))).collect()) + } + // Path D: top-N by BM25 with scores. + (true, Some(n)) => { + if n == 0 { + return Ok(Vec::new()); + } + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + let truncated = Self::sort_by_score_desc_truncate(filtered, n); + Ok(truncated.into_iter().map(|(s, id)| (id, Some(s))).collect()) + } + } + } + + /// Helper for paths B/C/D: run AllScoredCollector, translate doc_id → row_id, apply pre_filter. + /// Groups results by segment so the fast field column handle is opened once per segment + /// (same rationale as Path A — avoids per-match Column allocation). + fn collect_scored( + &self, + q: &dyn Query, + searcher: &tantivy::Searcher, + pre_filter: Option<&Treemap>, + ) -> Result, String> { + let scored = searcher + .search(q, &AllScoredCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + let mut by_segment: std::collections::HashMap> = + std::collections::HashMap::new(); + for (s, addr) in scored.into_iter() { + by_segment.entry(addr.segment_ord).or_default().push((s, addr.doc_id)); + } + let mut result: Vec<(Score, u64)> = Vec::new(); + for (segment_ord, entries) in by_segment.iter() { + let segment_reader = searcher.segment_reader(*segment_ord); + let fast = segment_reader + .fast_fields() + .u64(PAIMON_ROW_ID_FIELD_NAME) + .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}", + segment_ord))?; + for &(score, doc_id) in entries { + let rid = fast.first(doc_id).unwrap_or(0); + if pre_filter.map_or(true, |t| t.contains(rid)) { + result.push((score, rid)); + } + } + } + Ok(result) + } + + /// Helper for paths B/D: sort (score, row_id) by score desc with row_id asc tie-break, + /// then truncate to `n` items. + fn sort_by_score_desc_truncate(mut v: Vec<(Score, u64)>, n: usize) -> Vec<(Score, u64)> { + v.sort_unstable_by(|a, b| { + b.0.partial_cmp(&a.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.1.cmp(&b.1)) + }); + v.truncate(n); + v + } + + #[cfg(test)] + pub(crate) fn tokenizer_name(&self) -> &str { + &self.tokenizer_name + } + + #[cfg(test)] + pub(crate) fn debug_index(&self) -> &Index { + &self.index + } +} + +/// Escape regex metacharacters, but leave the input as a verbatim literal. +fn regex_escape(input: &str) -> String { + let mut out = String::with_capacity(input.len() + 4); + for ch in input.chars() { + match ch { + '.' | '+' | '*' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => { + out.push('\\'); + out.push(ch); + } + _ => out.push(ch), + } + } + out +} + +/// Normalize each literal segment independently while preserving wildcard +/// operators. This matches lucene-fts, where `*` and `?` are not analyzed. +fn normalize_wildcard_query(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut segment_start = 0; + for (index, ch) in input.char_indices() { + if ch != '*' && ch != '?' { + continue; + } + out.push_str(&normalize_case(&input[segment_start..index])); + out.push(ch); + segment_start = index + ch.len_utf8(); + } + out.push_str(&normalize_case(&input[segment_start..])); + out +} + +/// Translate a glob-style wildcard ('*' = any, '?' = single char) into a +/// regex pattern, escaping all other regex metacharacters. +fn wildcard_to_regex(input: &str) -> String { + let mut out = String::with_capacity(input.len() + 4); + for ch in input.chars() { + match ch { + '*' => out.push_str(".*"), + '?' => out.push('.'), + '.' | '+' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => { + out.push('\\'); + out.push(ch); + } + _ => out.push(ch), + } + } + out +} + +/// Collector that reads the explicit `row_id` u64 fast field directly into a +/// `Vec`, opening the column once per segment in `for_segment`. Replaces +/// the DocSetCollector → HashSet → per-doc translate path for unscored queries. +struct RowIdCollector; + +struct RowIdSegmentCollector { + row_id: Column, + ids: Vec, +} + +impl SegmentCollector for RowIdSegmentCollector { + type Fruit = Vec; + + fn collect(&mut self, doc: DocId, _score: Score) { + self.ids.push(self.row_id.first(doc).unwrap_or(0)); + } + + fn harvest(self) -> Vec { + self.ids + } +} + +impl Collector for RowIdCollector { + type Fruit = Vec; + type Child = RowIdSegmentCollector; + + fn for_segment( + &self, _ord: SegmentOrdinal, segment: &SegmentReader, + ) -> tantivy::Result { + let row_id = segment.fast_fields().u64(PAIMON_ROW_ID_FIELD_NAME)?; + Ok(RowIdSegmentCollector { row_id, ids: Vec::new() }) + } + + fn requires_scoring(&self) -> bool { + false + } + + fn merge_fruits(&self, segs: Vec>) -> tantivy::Result> { + Ok(segs.into_iter().flatten().collect()) + } +} + +/// Collector that returns at most `limit` DocAddresses across all segments, +/// no scoring. Shared atomic counter caps the global total so per-shard +/// transfer stays bounded for plain `LIMIT N` queries (no ORDER BY). +struct LimitedDocSetCollector { + limit: usize, + counter: std::sync::Arc, +} + +impl LimitedDocSetCollector { + fn new(limit: usize) -> Self { + Self { limit, counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)) } + } +} + +struct LimitedDocSetSegmentCollector { + segment_ord: SegmentOrdinal, + docs: Vec, + counter: std::sync::Arc, + limit: u64, +} + +impl SegmentCollector for LimitedDocSetSegmentCollector { + type Fruit = Vec; + + fn collect(&mut self, doc: DocId, _score: Score) { + // Best-effort cap: if multiple segments are scanned concurrently the + // atomic ensures we never accept more than `limit` rows total. + let prev = self.counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if prev < self.limit { + self.docs.push(doc); + } + } + + fn harvest(self) -> Self::Fruit { + let segment_ord = self.segment_ord; + self.docs.into_iter().map(|d| DocAddress::new(segment_ord, d)).collect() + } +} + +impl Collector for LimitedDocSetCollector { + type Fruit = Vec; + type Child = LimitedDocSetSegmentCollector; + + fn for_segment( + &self, segment_ord: SegmentOrdinal, _segment: &SegmentReader, + ) -> tantivy::Result { + Ok(LimitedDocSetSegmentCollector { + segment_ord, + docs: Vec::new(), + counter: self.counter.clone(), + limit: self.limit as u64, + }) + } + + fn requires_scoring(&self) -> bool { false } + + fn merge_fruits( + &self, segment_fruits: Vec>, + ) -> tantivy::Result> { + let mut result: Vec = segment_fruits.into_iter().flatten().collect(); + result.truncate(self.limit); + Ok(result) + } +} + +/// Custom Collector that returns ALL matching (score, DocAddress) tuples, +/// without truncation. tantivy's stock `TopDocs::with_limit(N)` would force +/// us to either pick N upfront (wrong when pre_filter rejects high-score +/// docs) or pass `usize::MAX` (which still enforces a binary heap on every +/// push). Our collector is just a plain Vec append, then merge. +struct AllScoredCollector; + +struct AllScoredSegmentCollector { + segment_ord: SegmentOrdinal, + docs: Vec<(Score, DocId)>, +} + +impl SegmentCollector for AllScoredSegmentCollector { + type Fruit = Vec<(Score, DocAddress)>; + + fn collect(&mut self, doc: DocId, score: Score) { + self.docs.push((score, doc)); + } + + fn harvest(self) -> Self::Fruit { + let segment_ord = self.segment_ord; + self.docs + .into_iter() + .map(|(s, d)| (s, DocAddress::new(segment_ord, d))) + .collect() + } +} + +impl Collector for AllScoredCollector { + type Fruit = Vec<(Score, DocAddress)>; + type Child = AllScoredSegmentCollector; + + fn for_segment( + &self, + segment_ord: SegmentOrdinal, + _segment: &SegmentReader, + ) -> tantivy::Result { + Ok(AllScoredSegmentCollector { + segment_ord, + docs: Vec::new(), + }) + } + + fn requires_scoring(&self) -> bool { + true + } + + fn merge_fruits( + &self, + segment_fruits: Vec>, + ) -> tantivy::Result> { + Ok(segment_fruits.into_iter().flatten().collect()) + } +} + +// ============================ FFI surface ============================ + +/// Construct a streaming reader from a layout table + pread callbacks. +/// +/// The layout arrays (names / offsets / lengths) are produced by C++-side +/// `ArchiveLayout::Parse` after reading only the archive header bytes. Payload +/// bytes are fetched lazily through `callbacks.read_at` as tantivy reads. +/// +/// # Arguments +/// * `file_names` — array of `file_count` UTF-8 NUL-terminated C strings +/// * `file_offsets` / `file_lengths` — u64 arrays (archive-absolute offsets and lengths) +/// * `file_count` — number of entries in each of the three arrays +/// * `callbacks` — pread + release callbacks; `ctx` ownership transfers to Rust +/// * `mode_cstr` — tokenize mode ("mp"/"mix"/"full"/"query"; "hmm" → Unsupported) +/// * `with_position` — whether text field was indexed with positions +/// * `dict_dir_cstr` — paimon_jieba dictionary directory +/// * `out` — receives the reader handle on success +/// +/// Releases an FFI stream ctx on drop unless disarmed. Gives +/// `paimon_tantivy_reader_new_streaming` a single ownership rule: Rust owns ctx +/// from entry and releases it on any error before the directory takes over. +struct StreamCtxReleaseGuard { + release: extern "C" fn(*mut std::os::raw::c_void), + ctx: *mut std::os::raw::c_void, + armed: bool, +} + +impl Drop for StreamCtxReleaseGuard { + fn drop(&mut self) { + if self.armed { + (self.release)(self.ctx); + } + } +} + +/// # Safety +/// All pointer args must be valid for the duration of the call. Ownership of +/// `callbacks.ctx` transfers to Rust on entry: on any error this function +/// releases it, and on success it lives until the reader handle is freed. The +/// caller must NOT release ctx itself after calling this function. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_new_streaming( + file_names: *const *const c_char, + file_offsets: *const u64, + file_lengths: *const u64, + file_count: usize, + callbacks: PaimonStreamCallbacks, + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + out: *mut *mut PaimonTantivyReader, +) -> PaimonTantivyStatus { + // Single, uniform ownership rule: Rust owns ctx from entry. This guard + // releases it on every error path until ownership moves into the directory + // (which then releases on its own drop). Prevents the C++ caller and Rust + // from both releasing the same ctx on the post-directory failure path. + let mut ctx_guard = StreamCtxReleaseGuard { + release: callbacks.release, + ctx: callbacks.ctx, + armed: true, + }; + + if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() { + set_last_error("paimon_tantivy_reader_new_streaming: null mandatory argument"); + return PaimonTantivyStatus::InvalidArgument; + } + if file_count > 0 + && (file_names.is_null() || file_offsets.is_null() || file_lengths.is_null()) + { + set_last_error("file_names/offsets/lengths must be non-null when file_count > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + + let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_str) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_str:?}; expected mp/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + + // Copy the C string array into owned Rust entries so the directory doesn't + // depend on caller-supplied lifetime. + let mut entries: Vec<(String, u64, u64)> = Vec::with_capacity(file_count); + for i in 0..file_count { + let name_ptr = unsafe { *file_names.add(i) }; + if name_ptr.is_null() { + set_last_error(format!("file_names[{i}] is null")); + return PaimonTantivyStatus::InvalidArgument; + } + let name = match unsafe { CStr::from_ptr(name_ptr) }.to_str() { + Ok(s) => s.to_owned(), + Err(e) => { + set_last_error(format!("file_names[{i}] not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let offset = unsafe { *file_offsets.add(i) }; + let length = unsafe { *file_lengths.add(i) }; + entries.push((name, offset, length)); + } + + // Ownership of ctx transfers to the directory from here on (it releases on + // its own drop, whether it fails below or lives inside the returned reader). + ctx_guard.armed = false; + let directory = PaimonCallbackDirectory::new(entries, callbacks); + + match PaimonTantivyReader::new(directory, mode, with_position, Path::new(dict_dir)) { + Ok(r) => { + unsafe { *out = into_handle(r) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let unsupported = e.contains("'hmm' is not supported"); + let bad_format = e.contains("tantivy::Index::open") + || e.contains("missing 'text' field"); + set_last_error(e); + if unsupported { + PaimonTantivyStatus::Unsupported + } else if bad_format { + PaimonTantivyStatus::IndexFormatError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Run a query and emit results into `out`. +/// +/// Output bytes (little-endian): +/// `[u8 has_scores | u64 count | u64 row_ids[count] | optional f32 scores[count]]` +/// +/// `has_scores=1` iff `limit >= 0` (caller asked for scoring + limit). +/// +/// `limit < 0` ⇒ no limit, no scoring; sorted ascending by row_id. +/// `limit >= 0` ⇒ top-N by descending score (pre_filter applied first). +/// `pre_filter_bytes`: serialized croaring `Roaring64Map::write` (portable), +/// containing paimon **row_ids** (not tantivy doc_ids); null+0 = no filter. +/// +/// SAFETY: `reader` must be a live handle; `query` and `pre_filter_bytes` +/// may be null+0 or readable slices; `out` non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_search( + reader: *mut PaimonTantivyReader, + search_type: i32, + query: *const c_char, + query_len: usize, + with_score: bool, + limit: i32, + pre_filter_bytes: *const c_char, + pre_filter_len: usize, + min_score: f32, + out: *mut PaimonTantivyBuffer, +) -> PaimonTantivyStatus { + if out.is_null() { + set_last_error("reader_search: out is null"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(r) = (unsafe { borrow_handle_mut::(reader) }) else { + set_last_error("reader_search: null reader handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + let st = match SearchType::from_i32(search_type) { + Some(s) => s, + None => { + set_last_error(format!("unknown search_type {search_type}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + if query.is_null() && query_len != 0 { + set_last_error("query is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let query_str = if query_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(query as *const u8, query_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("query not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + + let pre_filter: Option = if pre_filter_bytes.is_null() && pre_filter_len == 0 { + None + } else if pre_filter_bytes.is_null() { + set_last_error("pre_filter_bytes is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } else { + let slice = unsafe { + std::slice::from_raw_parts(pre_filter_bytes as *const u8, pre_filter_len) + }; + match Treemap::try_deserialize::(slice) { + Some(t) => Some(t), + None => { + set_last_error(format!( + "pre_filter not a valid Roaring64Map portable serialization ({} bytes)", + pre_filter_len + )); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + + let limit_opt: Option = if limit < 0 { None } else { Some(limit as usize) }; + let min_score_opt: Option = if min_score > 0.0 { Some(min_score) } else { None }; + + match r.search_with_limit_and_filter(st, query_str, with_score, limit_opt, pre_filter.as_ref(), min_score_opt) + { + Ok(rows) => { + // v0.2: has_scores is decoupled from limit — it equals with_score directly. + let has_scores = with_score; + let count = rows.len() as u64; + // 1 byte has_scores + 8 bytes count + 8 bytes per row_id + optional 4 bytes per score + let mut buf = Vec::with_capacity( + 1 + 8 + rows.len() * 8 + if has_scores { rows.len() * 4 } else { 0 }, + ); + buf.push(if has_scores { 1u8 } else { 0u8 }); + buf.extend_from_slice(&count.to_le_bytes()); + for (id, _) in &rows { + buf.extend_from_slice(&id.to_le_bytes()); // u64 row_id LE + } + if has_scores { + for (_, score) in &rows { + let s = score.unwrap_or(0.0); + buf.extend_from_slice(&s.to_le_bytes()); + } + } + unsafe { *out = PaimonTantivyBuffer::from_vec(buf) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let parse_err = e.contains("RegexQuery from") + || e.contains("phrase query") + || e.contains("produced no tokens"); + set_last_error(e); + if parse_err { + PaimonTantivyStatus::QueryParseError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Destroy a reader handle. Safe on null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_free(reader: *mut PaimonTantivyReader) { + unsafe { free_handle(reader) }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::callback_directory::test_support::{build_directory_from_archive, make_mock_callbacks}; + use crate::writer::PaimonTantivyWriter; + use std::path::PathBuf; + + // When PaimonTantivyReader::new fails *after* the callback directory (and + // thus ctx) is constructed, Rust must release ctx exactly once — never zero + // (leak) and never twice (the C++ caller no longer releases on failure, so a + // second release would be a double-free). file_count=0 yields an index with + // no meta.json, which fails to open after the directory is built. + #[test] + fn reader_new_streaming_releases_ctx_once_on_failure() { + let (cb, backend) = make_mock_callbacks(Vec::new()); + let mode = std::ffi::CString::new("mix").unwrap(); + let dict = std::ffi::CString::new("").unwrap(); + let mut out: *mut PaimonTantivyReader = std::ptr::null_mut(); + let st = unsafe { + paimon_tantivy_reader_new_streaming( + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + 0, + cb, + mode.as_ptr(), + true, + dict.as_ptr(), + &mut out, + ) + }; + assert!(!matches!(st, PaimonTantivyStatus::Ok)); + assert!(out.is_null()); + assert_eq!( + backend.release_count.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + } + + fn dict_dir() -> PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from("/tmp/nonexistent-dict")) + } + + fn build(docs: &[&str]) -> Vec { + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + for (i, d) in docs.iter().enumerate() { + w.add(i as u64, d).unwrap(); + } + w.finish().unwrap().1 + } + + fn open(packed: &[u8]) -> PaimonTantivyReader { + // Simulate production flow: parse archive header → build layout → + // back PaimonCallbackDirectory with a mock pread that reads from the + // packed Vec. Once C++ `ArchiveLayout::Parse` is in place, prod + // uses the same PaimonCallbackDirectory path. + let (dir, _backend) = build_directory_from_archive(packed.to_vec()); + PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap() + } + + #[test] + fn match_all_single_term() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "hello").unwrap(); + assert_eq!(ids, vec![0u64, 1]); + } + + #[test] + fn match_all_two_terms_intersection() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "hello world").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn match_any_two_terms_union() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAny, "hello peace").unwrap(); + assert_eq!(ids, vec![0u64, 1, 2]); + } + + #[test] + fn phrase_only_consecutive() { + let bytes = build(&["hello world there", "world hello there"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Phrase, "hello world").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn prefix_matches_indexed_terms() { + let bytes = build(&["unordered user-defined doc id"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Prefix, "unorder").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn prefix_normalizes_ascii_alphanumeric_case() { + let bytes = build(&["This is a test document", "another document"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Prefix, "THIS").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn wildcard_with_star() { + let bytes = build(&["unordered", "ordered", "border"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Wildcard, "*order*").unwrap(); + assert_eq!(ids, vec![0u64, 1, 2]); + } + + #[test] + fn wildcard_normalizes_ascii_alphanumeric_segments() { + let bytes = build(&["This is a test document", "another document"]); + let r = open(&bytes); + assert_eq!( + r.search_all(SearchType::Wildcard, "*THIS*").unwrap(), + vec![0u64] + ); + assert_eq!( + r.search_all(SearchType::Wildcard, "*?HIS*").unwrap(), + vec![0u64] + ); + } + + #[test] + fn prefix_and_wildcard_preserve_mixed_ascii_cjk_terms() { + let bytes = build(&["B超检查", "T恤"]); + let r = open(&bytes); + assert_eq!( + r.search_all(SearchType::Prefix, "B").unwrap(), + vec![0u64] + ); + assert_eq!( + r.search_all(SearchType::Wildcard, "*T*").unwrap(), + vec![1u64] + ); + } + + #[test] + fn empty_query_for_match_returns_query_parse_error() { + let bytes = build(&["hello"]); + let r = open(&bytes); + let err = r.search_all(SearchType::MatchAll, "").unwrap_err(); + assert!(err.contains("no tokens"), "got: {err}"); + } + + #[test] + fn wildcard_helper_escapes_dots() { + assert_eq!(wildcard_to_regex("a*b"), "a.*b"); + assert_eq!(wildcard_to_regex("a?b"), "a.b"); + assert_eq!(wildcard_to_regex("a.b"), r"a\.b"); + assert_eq!(wildcard_to_regex("*a*"), ".*a.*"); + } + + #[test] + fn wildcard_normalization_preserves_non_alphanumeric_segments() { + assert_eq!(normalize_wildcard_query("*?HIS*"), "*?his*"); + assert_eq!(normalize_wildcard_query("*THIS_IS*"), "*THIS_IS*"); + assert_eq!(normalize_wildcard_query("*机器*"), "*机器*"); + } + + // ----- limit + pre_filter + scoring (row_id-based) ----- + + #[test] + fn limit_returns_top_n_with_scores() { + let bytes = build(&[ + "doc", // 0: low score (1 occurrence) + "doc doc doc doc doc", // 1: high score (5 occurrences) + "doc doc", // 2: medium score + ]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(2), None, None) + .unwrap(); + assert_eq!(rows.len(), 2); + // doc 1 has highest TF, expect first + assert_eq!(rows[0].0, 1u64); + assert!(rows[0].1.is_some()); + assert!(rows[1].1.is_some()); + // Scores monotonically decreasing + assert!(rows[0].1.unwrap() >= rows[1].1.unwrap()); + } + + #[test] + fn no_limit_returns_all_unscored() { + let bytes = build(&["hello world", "world hello", "world peace"]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "world", false, None, None, None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![0u64, 1, 2]); + assert!(rows.iter().all(|(_, s)| s.is_none())); + } + + #[test] + fn pre_filter_no_limit_intersects() { + let bytes = build(&["alpha beta", "alpha gamma", "beta gamma"]); + let r = open(&bytes); + // pre_filter = {0, 2}; query "alpha" matches {0, 1}; expect intersection {0} + let mut tm = Treemap::new(); + tm.add(0); + tm.add(2); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn pre_filter_with_limit_filters_before_topn() { + // doc 0 has highest TF for "doc" but is NOT in pre_filter → must NOT + // be in result, even with limit=1. + let bytes = build(&[ + "doc doc doc doc doc", // 0: highest TF, but excluded + "doc doc", // 1: medium TF, included + "doc", // 2: low TF, excluded + ]); + let r = open(&bytes); + let mut tm = Treemap::new(); + tm.add(1); // only doc 1 passes pre_filter + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(10), Some(&tm), None) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0, 1u64); + } + + #[test] + fn unscored_limit_with_pre_filter_applies_filter_before_truncate() { + // Regression (review finding #1): with_score=false + limit=N + pre_filter + // must apply the filter to the FULL match set before truncating to N. + // All three docs match "doc" but only row_id 2 (the LAST one) passes the + // pre_filter; a truncate-before-filter impl (LimitedDocSetCollector that + // stops at N raw matches, then filters) would collect doc 0, filter it + // out, and wrongly return empty instead of {2}. + let bytes = build(&["doc", "doc", "doc"]); + let r = open(&bytes); + let mut tm = Treemap::new(); + tm.add(2); // only row_id 2 passes the pre_filter + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", false, Some(1), Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![2u64], "pre_filter must be applied before LIMIT truncation"); + assert!(rows.iter().all(|(_, s)| s.is_none())); + } + + #[test] + fn empty_pre_filter_returns_empty() { + let bytes = build(&["alpha", "beta"]); + let r = open(&bytes); + let tm = Treemap::new(); // empty + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + assert!(rows.is_empty()); + } + + #[test] + fn limit_zero_returns_empty_without_running_query() { + let bytes = build(&["alpha", "beta"]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", true, Some(0), None, None) + .unwrap(); + assert!(rows.is_empty()); + } + + // ----- row_id is independent of doc_id ----- + + #[test] + fn pre_filter_uses_row_id_not_doc_id() { + // Build with non-contiguous row_ids so doc_id ≠ row_id. Then verify + // pre_filter operates on row_id values, not internal tantivy doc_ids. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + w.add(100, "alpha").unwrap(); + w.add(200, "alpha").unwrap(); + w.add(300, "alpha").unwrap(); + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + + // pre_filter = {200} as row_id (doc_id would be 1) + let mut tm = Treemap::new(); + tm.add(200); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![200u64], "pre_filter must operate on row_id, not doc_id"); + } + + #[test] + fn search_returns_caller_supplied_row_ids() { + // Same setup: row_ids 100/200/300, verify search_all returns those values. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + w.add(100, "doc").unwrap(); + w.add(200, "doc").unwrap(); + w.add(300, "doc").unwrap(); + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "doc").unwrap(); + assert_eq!(ids, vec![100u64, 200, 300]); + } + + #[test] + fn tokenizer_name_reflects_paimon_jieba_schema_for_cpp_written_index() { + // cpp-written index: PaimonTantivyWriter binds the text field to + // `paimon_jieba`. Reader must pick that up from meta.json (not hardcode). + let bytes = build(&["hello world"]); + let r = open(&bytes); + assert_eq!(r.tokenizer_name(), PAIMON_TOKENIZER_NAME); + + // tokenize sanity: jieba mode="mix" picks `hello` + `world` from ASCII. + let q = r.tokenize_query("hello world"); + assert_eq!(q, vec!["hello".to_string(), "world".to_string()]); + } + + #[test] + fn tokenizer_name_reflects_default_schema_for_externally_written_index() { + // Simulate a paimon-java-shaped index: text field bound to the + // builtin `default` tokenizer (SimpleTokenizer + LowerCaser), not jieba. + // Build it directly via tantivy (bypassing PaimonTantivyWriter's jieba + // schema) so we can prove the reader auto-switches to the builtin. + use crate::callback_directory::test_support::build_mock_directory; + use tantivy::directory::Directory; + use tantivy::schema::{IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions}; + use tantivy::{doc, Index}; + + // Build a minimal index with field "text" bound to "default". + let mut sb = Schema::builder(); + let row_id_f = sb.add_u64_field( + "row_id", + NumericOptions::default().set_stored().set_indexed().set_fast(), + ); + let text_opts = TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("default") // ← key: match paimon-java's TEXT default + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ); + let text_f = sb.add_text_field("text", text_opts); + let schema = sb.build(); + let tmp = tempfile::Builder::new() + .prefix("paimon-tantivy-dyn-tk-") + .tempdir() + .unwrap(); + let index = Index::create_in_dir(tmp.path(), schema).unwrap(); + let mut writer = index.writer(15_000_000).unwrap(); + writer + .add_document(doc!(row_id_f => 0u64, text_f => "Hello World")) + .unwrap(); + writer + .add_document(doc!(row_id_f => 1u64, text_f => "Apple.Banana")) + .unwrap(); + writer.commit().unwrap(); + writer.wait_merging_threads().unwrap(); + + // Pack the index dir into our archive format so the callback directory + // can serve it. Reuse writer.rs's format by streaming entries manually. + let mut data = Vec::new(); + let mut entries = Vec::<(String, u64, u64)>::new(); + let dir_iter = std::fs::read_dir(tmp.path()).unwrap(); + let mut files: Vec<_> = dir_iter + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().ok().map_or(false, |t| t.is_file())) + .filter(|e| !e.file_name().to_string_lossy().starts_with('.')) + .collect(); + files.sort_by_key(|e| e.file_name()); + data.extend_from_slice(&(files.len() as i32).to_be_bytes()); + for e in &files { + let name = e.file_name().to_string_lossy().into_owned(); + let bytes = std::fs::read(e.path()).unwrap(); + data.extend_from_slice(&(name.len() as i32).to_be_bytes()); + data.extend_from_slice(name.as_bytes()); + data.extend_from_slice(&(bytes.len() as i64).to_be_bytes()); + let off = data.len() as u64; + data.extend_from_slice(&bytes); + entries.push((name, off, bytes.len() as u64)); + } + + let (dir, _backend) = build_mock_directory(data, entries); + let r = PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap(); + + // Reader must pick up `default` from schema, not hardcode `paimon_jieba`. + assert_eq!(r.tokenizer_name(), "default"); + + // Query tokenization now goes through tantivy's builtin default + // (SimpleTokenizer + LowerCaser): + // "Apple.Banana" → ["apple", "banana"] (dot is non-alnum, split) + // "Hello World" → ["hello", "world"] (space split + lowercase) + let q1 = r.tokenize_query("Hello World"); + assert_eq!(q1, vec!["hello".to_string(), "world".to_string()]); + let q2 = r.tokenize_query("Apple.Banana"); + assert_eq!(q2, vec!["apple".to_string(), "banana".to_string()]); + + // And the search path works across tokenizer: + let ids = r.search_all(SearchType::MatchAll, "hello").unwrap(); + assert_eq!(ids, vec![0u64]); + let ids = r.search_all(SearchType::MatchAll, "apple").unwrap(); + assert_eq!(ids, vec![1u64]); + } + + #[test] + fn reader_aggregates_row_ids_across_segments() { + // Multi-thread default writer + many docs => may produce multiple + // segments before force-merge. After finish(), force-merge collapses + // to one segment, but this test still validates the row_id retrieval + // path works for ≥1 segment. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i * 7, &format!("docmark_{i} apple")).unwrap(); + } + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "apple").unwrap(); + assert_eq!(ids.len(), 200); + for i in 0..200u64 { + assert!(ids.contains(&(i * 7)), "missing row_id={}", i * 7); + } + } +} diff --git a/crates/tantivy_ffi/src/tokenizer.rs b/crates/tantivy_ffi/src/tokenizer.rs new file mode 100644 index 00000000..19139fcb --- /dev/null +++ b/crates/tantivy_ffi/src/tokenizer.rs @@ -0,0 +1,467 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PaimonJiebaTokenizer: tantivy Tokenizer impl wrapping jieba-rs. +//! +//! Contract: +//! - Behavior-equivalent with `JiebaAnalyzer` in src/paimon/global_index/lucene/ +//! - 5 modes: mp / hmm / mix / full / query +//! - `hmm` is Unsupported (jieba-rs has no standalone HMM entry point) +//! - `mp` accepts cut(hmm=false) but does not replicate cppjieba's +//! max_word_len truncation +//! - Normalize: skip pure whitespace, skip stop_words, lowercase ASCII-only tokens +//! - Token offsets: byte offsets into the original UTF-8 string +//! - `with_position=false`: all tokens emitted at `position=0` (disables PhraseQuery) +//! - Custom dict dir: loads `jieba.dict.utf8` (+optional `user.dict.utf8`) from +//! `$PAIMON_JIEBA_DICT_DIR`; stop_words.utf8 loaded if present + +use std::collections::HashSet; +use std::ffi::{c_char, CStr}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::Arc; + +use jieba_rs::Jieba; +use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; + +use crate::buffer::PaimonTantivyBuffer; +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle, free_handle, into_handle}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TokenizeMode { + Mp, + Hmm, + Mix, + Full, + Query, +} + +impl TokenizeMode { + pub(crate) fn parse(s: &str) -> Option { + match s { + "mp" => Some(Self::Mp), + "hmm" => Some(Self::Hmm), + "mix" => Some(Self::Mix), + "full" => Some(Self::Full), + "query" => Some(Self::Query), + _ => None, + } + } +} + +#[derive(Clone)] +pub struct PaimonJiebaTokenizer { + jieba: Arc, + mode: TokenizeMode, + with_position: bool, + stop_words: Arc>, +} + +impl PaimonJiebaTokenizer { + pub fn new( + dict_dir: &Path, + mode: TokenizeMode, + with_position: bool, + ) -> Result { + if mode == TokenizeMode::Hmm { + return Err( + "tokenize mode 'hmm' is not supported (jieba-rs does not expose standalone HMM)" + .into(), + ); + } + let jieba = load_jieba(dict_dir)?; + let stop_words = load_stop_words(dict_dir); + Ok(Self { + jieba: Arc::new(jieba), + mode, + with_position, + stop_words: Arc::new(stop_words), + }) + } + + /// Directly tokenize, returning a Vec of (offset_start, offset_end, text) tuples. + /// Used both by the tantivy Tokenizer impl and the standalone `tokenize` FFI. + pub fn tokenize_raw(&self, text: &str) -> Vec<(usize, usize, String)> { + // Use jieba-rs's cut variants which return Vec<&'a str>; compute byte offsets + // via pointer arithmetic (each &str is a slice of the original). + let cuts: Vec<&str> = match self.mode { + TokenizeMode::Mp => self.jieba.cut(text, false), + TokenizeMode::Hmm => Vec::new(), // unreachable (caught in new()) + TokenizeMode::Mix => self.jieba.cut(text, true), + TokenizeMode::Full => self.jieba.cut_all(text), + TokenizeMode::Query => self.jieba.cut_for_search(text, true), + }; + + let text_start = text.as_ptr() as usize; + let mut out = Vec::with_capacity(cuts.len()); + for piece in cuts { + // skip pure whitespace + if piece.chars().all(char::is_whitespace) { + continue; + } + // skip stop words (compare original case) + if self.stop_words.contains(piece) { + continue; + } + // offset calc + let start = piece.as_ptr() as usize - text_start; + let end = start + piece.len(); + // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior) + let token_text = normalize_case(piece); + out.push((start, end, token_text)); + } + out + } +} + +fn is_ascii_alnum(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric()) +} + +pub(crate) fn normalize_case(s: &str) -> String { + if is_ascii_alnum(s) { + s.to_ascii_lowercase() + } else { + s.to_string() + } +} + +fn load_jieba(dict_dir: &Path) -> Result { + let main_dict = dict_dir.join("jieba.dict.utf8"); + let mut jieba = if main_dict.exists() { + let file = File::open(&main_dict) + .map_err(|e| format!("open {}: {e}", main_dict.display()))?; + let mut rdr = BufReader::new(file); + Jieba::with_dict(&mut rdr).map_err(|e| format!("load jieba dict: {e:?}"))? + } else { + // No custom dict; use jieba-rs builtin + Jieba::new() + }; + // Optional user dict. cppjieba's user.dict.utf8 is lenient: lines are + // `word [freq] [tag]` where freq can be omitted (e.g. ` nz`), but + // jieba-rs's load_dict strictly requires `word freq [tag]` and fails if + // freq is not an integer. We parse line-by-line with `add_word` to stay + // compatible. + let user_dict = dict_dir.join("user.dict.utf8"); + if user_dict.exists() { + let file = File::open(&user_dict) + .map_err(|e| format!("open {}: {e}", user_dict.display()))?; + for (n, line_res) in BufReader::new(file).lines().enumerate() { + let line = match line_res { + Ok(l) => l, + Err(_) => continue, // skip unreadable lines + }; + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let mut it = trimmed.split_whitespace(); + let word = it.next().unwrap(); // non-empty guaranteed + let next = it.next(); + let freq = next.and_then(|s| s.parse::().ok()); + let tag = match (freq, next) { + (Some(_), _) => it.next(), // [tag] + (None, tok) => tok, // (no freq) + }; + // `add_word` returns the assigned frequency; ignore it. For lines + // with bogus content we silently keep going, matching cppjieba's + // tolerant behavior. + let _ = jieba.add_word(word, freq, tag); + let _ = n; // keep for potential debug + } + } + Ok(jieba) +} + +fn load_stop_words(dict_dir: &Path) -> HashSet { + let path = dict_dir.join("stop_words.utf8"); + let mut out = HashSet::new(); + if let Ok(f) = File::open(&path) { + for line in BufReader::new(f).lines().map_while(Result::ok) { + let w = line.trim(); + if !w.is_empty() { + out.insert(w.to_owned()); + } + } + } + out +} + +// ----------------- tantivy Tokenizer integration ----------------- + +pub struct PaimonJiebaTokenStream { + tokens: Vec, + index: usize, +} + +impl TokenStream for PaimonJiebaTokenStream { + fn advance(&mut self) -> bool { + self.index += 1; + self.index <= self.tokens.len() + } + + fn token(&self) -> &Token { + &self.tokens[self.index - 1] + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.tokens[self.index - 1] + } +} + +impl Tokenizer for PaimonJiebaTokenizer { + type TokenStream<'a> = PaimonJiebaTokenStream; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + let raw = self.tokenize_raw(text); + let tokens: Vec = raw + .into_iter() + .enumerate() + .map(|(i, (s, e, t))| Token { + offset_from: s, + offset_to: e, + position: if self.with_position { i } else { 0 }, + text: t, + position_length: 1, + }) + .collect(); + PaimonJiebaTokenStream { tokens, index: 0 } + } +} + +// ----------------- FFI surface ----------------- + +/// Create a tokenizer handle. Returns OK and writes *out on success; returns +/// status and sets last_error on failure. +/// +/// SAFETY: `mode_cstr` and `dict_dir_cstr` must be NUL-terminated UTF-8; +/// `out` must be a valid non-null pointer. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_new( + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + out: *mut *mut PaimonJiebaTokenizer, +) -> PaimonTantivyStatus { + if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() { + set_last_error("paimon_tantivy_tokenizer_new: null argument"); + return PaimonTantivyStatus::InvalidArgument; + } + let mode_s = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_s = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_s) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_s:?}; expected one of mp/hmm/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + match PaimonJiebaTokenizer::new(Path::new(dict_s), mode, with_position) { + Ok(t) => { + unsafe { *out = into_handle(t) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let is_hmm_unsupported = e.contains("'hmm' is not supported"); + set_last_error(e); + if is_hmm_unsupported { + PaimonTantivyStatus::Unsupported + } else { + PaimonTantivyStatus::TokenizerError + } + } + } +} + +/// Free a tokenizer handle. Safe on null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_free(tok: *mut PaimonJiebaTokenizer) { + unsafe { free_handle(tok) }; +} + +/// Tokenize a string and return a newline-delimited list of tokens as bytes. +/// Used for golden-sample tests (easy to diff from C++). +/// +/// Output format: +/// `\t\t\t\n` for each token. +/// +/// SAFETY: `tok` must be a valid handle; `text` must point to `text_len` UTF-8 bytes; +/// `out` must be non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_tokenize( + tok: *const PaimonJiebaTokenizer, + text: *const c_char, + text_len: usize, + out: *mut PaimonTantivyBuffer, +) -> PaimonTantivyStatus { + if out.is_null() { + set_last_error("paimon_tantivy_tokenizer_tokenize: out is null"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(tokenizer) = (unsafe { borrow_handle::(tok) }) else { + set_last_error("paimon_tantivy_tokenizer_tokenize: null tokenizer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + if text.is_null() && text_len != 0 { + set_last_error("text is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let text_str = if text_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("text not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + let raw = tokenizer.tokenize_raw(text_str); + let mut buf = String::new(); + for (i, (s, e, t)) in raw.iter().enumerate() { + let pos = if tokenizer.with_position { i } else { 0 }; + buf.push_str(&format!("{s}\t{e}\t{pos}\t{t}\n")); + } + let bytes = buf.into_bytes(); + unsafe { *out = PaimonTantivyBuffer::from_vec(bytes) }; + PaimonTantivyStatus::Ok +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + fn dict_dir_from_env() -> std::path::PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict")) + } + + #[test] + fn mode_parse() { + for (s, m) in [ + ("mp", TokenizeMode::Mp), + ("hmm", TokenizeMode::Hmm), + ("mix", TokenizeMode::Mix), + ("full", TokenizeMode::Full), + ("query", TokenizeMode::Query), + ] { + assert_eq!(TokenizeMode::parse(s), Some(m)); + } + assert!(TokenizeMode::parse("bogus").is_none()); + } + + #[test] + fn hmm_mode_returns_unsupported() { + let tok = PaimonJiebaTokenizer::new( + &dict_dir_from_env(), + TokenizeMode::Hmm, + true, + ); + match tok { + Err(e) => assert!(e.contains("'hmm' is not supported"), "got: {e}"), + Ok(_) => panic!("expected Err"), + } + } + + #[test] + fn tokenize_mix_default_dict_smoke() { + // If no custom dict dir, jieba-rs builtin is used. + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nonexistent-dict"), TokenizeMode::Mix, true) + .unwrap(); + let raw = t.tokenize_raw("他来到了网易杭研大厦"); + let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect(); + assert!(texts.contains(&"网易")); + assert!(texts.contains(&"大厦")); + } + + #[test] + fn ascii_alnum_is_lowercased() { + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, true).unwrap(); + let raw = t.tokenize_raw("Hello World 中国"); + let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect(); + assert!(texts.contains(&"hello")); + assert!(texts.contains(&"world")); + assert!(texts.contains(&"中国")); + } + + #[test] + fn with_position_false_emits_zero_position() { + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, false).unwrap(); + let raw = t.tokenize_raw("中国人"); + // Can't check position on raw tuples; check via tantivy Token stream: + let mut t2 = t.clone(); + let mut stream = ::token_stream(&mut t2, "中国人"); + let mut positions = Vec::new(); + while stream.advance() { + positions.push(stream.token().position); + } + assert!(!raw.is_empty()); + assert!(positions.iter().all(|&p| p == 0)); + } + + #[test] + fn ffi_roundtrip() { + let dict = dict_dir_from_env(); + let dict_str = dict.to_str().unwrap(); + let mode = CString::new("mix").unwrap(); + let dict_c = CString::new(dict_str).unwrap(); + let mut handle: *mut PaimonJiebaTokenizer = std::ptr::null_mut(); + unsafe { + let st = paimon_tantivy_tokenizer_new( + mode.as_ptr(), + true, + dict_c.as_ptr(), + &mut handle, + ); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert!(!handle.is_null()); + + let input = "Hello 中国"; + let input_c = CString::new(input).unwrap(); + let mut buf = PaimonTantivyBuffer::empty(); + let st2 = paimon_tantivy_tokenizer_tokenize( + handle, + input_c.as_ptr(), + input.len(), + &mut buf, + ); + assert_eq!(st2, PaimonTantivyStatus::Ok); + assert!(buf.len > 0); + crate::buffer::paimon_tantivy_buffer_free(&mut buf); + paimon_tantivy_tokenizer_free(handle); + } + } +} diff --git a/crates/tantivy_ffi/src/writer.rs b/crates/tantivy_ffi/src/writer.rs new file mode 100644 index 00000000..1a068e7b --- /dev/null +++ b/crates/tantivy_ffi/src/writer.rs @@ -0,0 +1,788 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PaimonTantivyWriter: Writer for tantivy-fulltext global index. +//! +//! Contract: +//! - `writer_new(field_name, mode, with_position, dict_dir, out)` — create on a +//! private tmp dir backed by MmapDirectory + PaimonJiebaTokenizer. +//! `field_name` is **ignored** by the Rust schema (kept for FFI ABI +//! compatibility); schema field names are fixed (`row_id`, `text`) to match +//! paimon-java `paimon-tantivy-jni/rust/src/lib.rs:55-66`. +//! - `writer_add(writer, row_id, text, len)` — add a single document with the +//! caller-supplied `row_id` (u64) and a TEXT field +//! - `writer_finish(writer, out_row_count, out_buf)` — commit + force-merge to +//! single segment + pack all on-disk index files into a Rust-allocated buffer +//! - `writer_free(writer)` — destroy (RAII removes tmp dir) +//! +//! Packing format (big-endian, **cross-readable with paimon-java archives**): +//! `[i32 BE file_count | +//! (i32 BE name_len | name_bytes | i64 BE file_len | file_bytes)*]` + +use std::ffi::{c_char, c_void, CStr}; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use tantivy::schema::{ + Field, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, +}; +use tantivy::{doc, Index, IndexWriter, TantivyDocument}; +use tempfile::TempDir; + +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle_mut, free_handle, into_handle}; +use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode}; + +/// Schema field names. Fixed to match paimon-java's tantivy schema so that +/// indexes are cross-readable. Both fields are required. +pub const PAIMON_ROW_ID_FIELD_NAME: &str = "row_id"; +pub const PAIMON_TEXT_FIELD_NAME: &str = "text"; + +/// Name registered with the tantivy `TokenizerManager`. Reader must register +/// the same name to make stored term dictionaries readable. +pub const PAIMON_TOKENIZER_NAME: &str = "paimon_jieba"; + +/// Heap budget for the in-process IndexWriter (50 MB; tantivy minimum is ~3 MB). +/// Default multi-threaded writer (`Index::writer(heap)`) splits this budget +/// across `min(num_cpus, MAX_NUM_THREAD=8)` worker threads. +const WRITER_HEAP_SIZE: usize = 50_000_000; + +pub struct PaimonTantivyWriter { + /// Owned tmp dir; cleaned up when this struct drops. + tmpdir: TempDir, + /// `row_id` u64 field (stored + indexed + fast). Reader retrieves the + /// caller-supplied row_id via `fast_fields().u64("row_id").first(doc_id)`. + row_id_field: Field, + /// `text` TEXT field tokenized via the registered jieba tokenizer. + text_field: Field, + /// tantivy index instance, file-backed in `tmpdir`. + index: Index, + /// Active writer; consumed by `wait_merging_threads()` in `finish`. + writer: Option, + /// Documents added since construction. + row_count: i64, +} + +impl PaimonTantivyWriter { + pub fn new( + field_name: &str, + mode: TokenizeMode, + with_position: bool, + dict_dir: &Path, + tokenizer_name: &str, + ) -> Result { + if field_name.is_empty() { + return Err("field_name must be non-empty".into()); + } + // Schema is fixed to match paimon-java: row_id (u64 + // stored+indexed+fast) + text (TEXT). The caller-supplied `field_name` + // parameter is currently ignored by the Rust schema (kept for FFI + // backward-compatibility); the C++ side still uses it to extract the + // right column from arrow batches. + let _ = field_name; // intentionally unused on the Rust side + let mut schema_builder = Schema::builder(); + let row_id_field = schema_builder.add_u64_field( + PAIMON_ROW_ID_FIELD_NAME, + NumericOptions::default() + .set_stored() + .set_indexed() + .set_fast(), + ); + let index_option = if with_position { + IndexRecordOption::WithFreqsAndPositions + } else { + IndexRecordOption::Basic + }; + // Empty input falls back to tantivy's built-in "default" (SimpleTokenizer), + // matching the cpp-side default in `tantivy_defs.h::kDefaultTantivyWriteTokenizer`. + // Cross-read with paimon-java works out of the box; CJK callers must + // pass "paimon_jieba" explicitly. + let effective_tokenizer = if tokenizer_name.is_empty() { + "default" + } else { + tokenizer_name + }; + let text_options = TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(effective_tokenizer) + .set_index_option(index_option), + ); + let text_field = schema_builder.add_text_field(PAIMON_TEXT_FIELD_NAME, text_options); + let schema = schema_builder.build(); + + let tmpdir = tempfile::Builder::new() + .prefix("paimon-tantivy-") + .tempdir() + .map_err(|e| format!("create tmp dir: {e}"))?; + + let index = Index::create_in_dir(tmpdir.path(), schema) + .map_err(|e| format!("create tantivy index: {e}"))?; + // When caller picks "paimon_jieba" we construct + register the jieba + // tokenizer. For any tantivy built-in name ("default", "whitespace", + // "raw", "en_stem", ...) tantivy's TokenizerManager already has it + // registered via `TokenizerManager::default()`; no-op here. This lets + // paimon-cpp emit archives cross-readable by paimon-java's default + // TEXT tokenizer path. + if effective_tokenizer == PAIMON_TOKENIZER_NAME { + let tokenizer = PaimonJiebaTokenizer::new(dict_dir, mode, with_position) + .map_err(|e| format!("create tokenizer: {e}"))?; + index + .tokenizers() + .register(PAIMON_TOKENIZER_NAME, tokenizer); + } + + // Default multi-threaded writer (schema stores row_id explicitly so + // we no longer need single-threaded ordering invariants). tantivy will + // use min(num_cpus, MAX_NUM_THREAD=8) workers, splitting heap budget. + let writer: IndexWriter = index + .writer(WRITER_HEAP_SIZE) + .map_err(|e| format!("create index writer: {e}"))?; + + Ok(Self { + tmpdir, + row_id_field, + text_field, + index, + writer: Some(writer), + row_count: 0, + }) + } + + pub fn add(&mut self, row_id: u64, text: &str) -> Result<(), String> { + let writer = self + .writer + .as_mut() + .ok_or_else(|| "writer already finished".to_string())?; + let document: TantivyDocument = doc!( + self.row_id_field => row_id, + self.text_field => text, + ); + writer + .add_document(document) + .map_err(|e| format!("add document: {e}"))?; + self.row_count += 1; + Ok(()) + } + + /// Commit + force-merge + GC on-disk index. Extracted from `finish_*` + /// so both streaming and test paths can share it. + fn commit_and_merge(&mut self) -> Result<(), String> { + let mut writer = self + .writer + .take() + .ok_or_else(|| "writer already finished".to_string())?; + writer.commit().map_err(|e| format!("commit: {e}"))?; + + let segment_metas = self + .index + .searchable_segment_metas() + .map_err(|e| format!("list segments: {e}"))?; + if segment_metas.len() > 1 { + let segment_ids: Vec<_> = segment_metas.iter().map(|m| m.id()).collect(); + writer + .merge(&segment_ids) + .wait() + .map_err(|e| format!("merge: {e}"))?; + } + writer + .garbage_collect_files() + .wait() + .map_err(|e| format!("garbage_collect_files: {e}"))?; + writer + .wait_merging_threads() + .map_err(|e| format!("wait_merging_threads: {e}"))?; + Ok(()) + } + + /// Streaming finish (production path): commit + force-merge + push archive + /// bytes through the FFI callback in fixed-size chunks (see + /// WRITER_STREAM_BUFFER_SIZE). Peak RAM independent of archive size — one + /// heap buffer + a few KB metadata. + pub fn finish_streaming( + &mut self, + cb: &PaimonWriteCallbacks, + ) -> Result { + self.commit_and_merge()?; + let ctx = cb.ctx; + let write_fn = cb.write; + pack_index_dir_stream(self.tmpdir.path(), |bytes| { + // Calling extern "C" fn pointer is safe; C++ side owns ctx validity. + let rc = (write_fn)(ctx, bytes.as_ptr(), bytes.len()); + if rc != 0 { + return Err(format!("write callback rc={rc} len={}", bytes.len())); + } + Ok(()) + })?; + Ok(self.row_count) + } + + /// Test-only convenience: collect streaming output into a `Vec`. + /// Rust unit tests / integration tests use this; production path is + /// `finish_streaming`. + #[cfg(test)] + pub(crate) fn finish(&mut self) -> Result<(i64, Vec), String> { + self.commit_and_merge()?; + let mut out: Vec = Vec::new(); + pack_index_dir_stream(self.tmpdir.path(), |bytes| { + out.extend_from_slice(bytes); + Ok(()) + })?; + Ok((self.row_count, out)) + } + + #[cfg(test)] + pub(crate) fn tmpdir_path(&self) -> &Path { + self.tmpdir.path() + } +} + +// ========================================================================= +// Streaming pack +// ========================================================================= + +/// Streaming pack buffer size. 1MB matches the buffer size data-lake storage +/// backends (e.g. Pangu) use for good throughput, still far below any archive +/// size we care about. Heap-allocated (see pack_index_dir_stream), so the size +/// does not affect stack usage. +const WRITER_STREAM_BUFFER_SIZE: usize = 1024 * 1024; + +/// Callback table passed from C++ for streaming writer output. +/// +/// `ctx` is an opaque pointer to C++'s `WriteCtx` (holding a `paimon::OutputStream`). +/// `write` is called in-order by Rust (not concurrently) to push bytes. +#[repr(C)] +pub struct PaimonWriteCallbacks { + pub ctx: *mut c_void, + /// Returns 0 on success, non-zero to signal C++ side error (Rust aborts pack). + pub write: extern "C" fn(ctx: *mut c_void, data: *const u8, len: usize) -> i32, +} + +/// Walk tempdir + pack into the Java-compatible archive format, pushing each +/// chunk through `write_fn`. Peak RAM = one WRITER_STREAM_BUFFER_SIZE heap +/// buffer + a few KB of entry metadata (name + PathBuf + u64 length). Mirrors +/// Java `TantivyFullTextGlobalIndexWriter.packIndex` but with a bigger buffer. +/// +/// Archive format (BE, no version): `[i32 file_count | (i32 name_len, name, +/// i64 file_len, file_bytes)*]`. Files sorted alphabetically for deterministic +/// output; `.`-prefixed (lock) files and non-regular entries skipped. +fn pack_index_dir_stream(dir: &Path, mut write_fn: F) -> Result<(), String> +where + F: FnMut(&[u8]) -> Result<(), String>, +{ + let entries = collect_dir_entries(dir)?; + + // Header: BE i32 file_count + write_fn(&(entries.len() as i32).to_be_bytes())?; + + let mut buf = vec![0u8; WRITER_STREAM_BUFFER_SIZE]; + for (name, path, file_len) in &entries { + // Per-entry header: name_len, name, data_len + write_fn(&(name.len() as i32).to_be_bytes())?; + write_fn(name.as_bytes())?; + write_fn(&(*file_len as i64).to_be_bytes())?; + + // Payload: fixed-size buffer loop + let mut f = File::open(path) + .map_err(|e| format!("open {}: {e}", path.display()))?; + let mut pushed: u64 = 0; + loop { + let n = f + .read(&mut buf) + .map_err(|e| format!("read {}: {e}", path.display()))?; + if n == 0 { + break; + } + write_fn(&buf[..n])?; + pushed += n as u64; + } + if pushed != *file_len { + return Err(format!( + "file {} changed size during packing: header said {}, streamed {}", + name, file_len, pushed + )); + } + } + Ok(()) +} + +/// Enumerate the tempdir: sorted (name, path, len) for regular non-`.lock` files. +fn collect_dir_entries(dir: &Path) -> Result, String> { + let mut entries: Vec<(String, PathBuf, u64)> = Vec::new(); + let read_dir = + std::fs::read_dir(dir).map_err(|e| format!("read tmp dir {}: {e}", dir.display()))?; + for entry_res in read_dir { + let entry = entry_res.map_err(|e| format!("read entry: {e}"))?; + let name = match entry.file_name().into_string() { + Ok(n) => n, + Err(_) => continue, + }; + if name.starts_with('.') { + continue; + } + let ft = entry + .file_type() + .map_err(|e| format!("file_type for {}: {e}", entry.path().display()))?; + if !ft.is_file() { + continue; + } + let len = entry + .metadata() + .map_err(|e| format!("metadata for {}: {e}", entry.path().display()))? + .len(); + entries.push((name, entry.path(), len)); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(entries) +} + +// ============================ FFI surface ============================ + +/// Create a writer handle on a private tmp dir. +/// +/// SAFETY: all C-string args must be NUL-terminated UTF-8; `out` non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_new( + field_name_cstr: *const c_char, + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + tokenizer_cstr: *const c_char, + out: *mut *mut PaimonTantivyWriter, +) -> PaimonTantivyStatus { + if field_name_cstr.is_null() + || mode_cstr.is_null() + || dict_dir_cstr.is_null() + || tokenizer_cstr.is_null() + || out.is_null() + { + set_last_error("paimon_tantivy_writer_new: null argument"); + return PaimonTantivyStatus::InvalidArgument; + } + let field_name = match unsafe { CStr::from_ptr(field_name_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("field_name not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let tokenizer_name = match unsafe { CStr::from_ptr(tokenizer_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("tokenizer not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_str) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_str:?}; expected one of mp/hmm/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + match PaimonTantivyWriter::new( + field_name, + mode, + with_position, + Path::new(dict_dir), + tokenizer_name, + ) { + Ok(w) => { + unsafe { *out = into_handle(w) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + // hmm-mode rejection bubbles through tokenizer construction. + let unsupported = e.contains("'hmm' is not supported"); + set_last_error(e); + if unsupported { + PaimonTantivyStatus::Unsupported + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Add a single document. `text` need not be NUL-terminated; treat as a slice +/// of `text_len` UTF-8 bytes. Empty text (len=0) inserts an empty-text doc. +/// `row_id` is the caller-supplied paimon row id (u64), stored in a fast field +/// for retrieval by the reader. +/// +/// SAFETY: `writer` must be a live handle from `writer_new`. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_add( + writer: *mut PaimonTantivyWriter, + row_id: u64, + text: *const c_char, + text_len: usize, +) -> PaimonTantivyStatus { + let Some(w) = (unsafe { borrow_handle_mut::(writer) }) else { + set_last_error("paimon_tantivy_writer_add: null writer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + if text.is_null() && text_len != 0 { + set_last_error("text is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let text_str = if text_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("text not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + match w.add(row_id, text_str) { + Ok(()) => PaimonTantivyStatus::Ok, + Err(e) => { + set_last_error(e); + PaimonTantivyStatus::InternalError + } + } +} + +/// Commit + force-merge + stream archive bytes through `callbacks.write` in +/// fixed-size chunks (see WRITER_STREAM_BUFFER_SIZE). May only be called once +/// per writer; subsequent calls return InvalidArgument with +/// last_error="writer already finished". Peak Rust RAM ≈ one buffer + entry +/// metadata (independent of archive size). +/// +/// The callback is invoked **serially** (not concurrently) within this call; +/// C++ side can write directly to paimon OutputStream without locking. +/// +/// SAFETY: `writer` must be a live handle; `out_row_count` non-null. +/// `callbacks.write` / `callbacks.ctx` must remain valid for the duration of +/// the call (callback is consumed in-place, not retained). +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_finish_streaming( + writer: *mut PaimonTantivyWriter, + callbacks: PaimonWriteCallbacks, + out_row_count: *mut i64, +) -> PaimonTantivyStatus { + if out_row_count.is_null() { + set_last_error("paimon_tantivy_writer_finish_streaming: null out_row_count"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(w) = (unsafe { borrow_handle_mut::(writer) }) else { + set_last_error("paimon_tantivy_writer_finish_streaming: null writer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + match w.finish_streaming(&callbacks) { + Ok(rows) => { + unsafe { *out_row_count = rows }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let already_finished = e == "writer already finished"; + let io_err = e.starts_with("write callback rc=") + || e.starts_with("open ") + || e.starts_with("read "); + set_last_error(e); + if already_finished { + PaimonTantivyStatus::InvalidArgument + } else if io_err { + PaimonTantivyStatus::IoError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Destroy a writer handle. Safe on null. Tmp dir is removed via Drop. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_free(writer: *mut PaimonTantivyWriter) { + unsafe { free_handle(writer) }; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + /// Test dict dir for jieba; defaults to a non-existent path so jieba-rs uses + /// its built-in dict (which is enough for these smoke tests). + fn dict_dir_from_env() -> std::path::PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict")) + } + + #[test] + fn empty_field_name_rejected() { + let err = PaimonTantivyWriter::new("", TokenizeMode::Mix, true, Path::new("/tmp/nx"), "paimon_jieba") + .err() + .unwrap(); + assert!(err.contains("field_name"), "got: {err}"); + } + + #[test] + fn hmm_mode_rejected() { + let err = + PaimonTantivyWriter::new("f0", TokenizeMode::Hmm, true, Path::new("/tmp/nx"), "paimon_jieba") + .err() + .unwrap(); + assert!(err.contains("'hmm' is not supported"), "got: {err}"); + } + + #[test] + fn create_add_finish_roundtrip() { + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello world").unwrap(); + w.add(1, "中国人民").unwrap(); + w.add(2, "").unwrap(); // empty doc + let (rows, bytes) = w.finish().unwrap(); + assert_eq!(rows, 3); + assert!(bytes.len() > 4); + + // Validate header (Java-compatible: BE int32 file_count, no version) + let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + assert!(file_count > 0, "expected >0 packed files"); + + // Walk entries (BE) + let mut off: usize = 4; + let mut names = Vec::new(); + for _ in 0..file_count { + let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let name = std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned(); + off += nlen; + let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as usize; + off += 8; + assert!(off + flen <= bytes.len(), "file {name} extends past buffer"); + off += flen; + names.push(name); + } + assert_eq!(off, bytes.len(), "trailing bytes after pack"); + // tantivy must produce at least meta.json + assert!(names.iter().any(|n| n == "meta.json"), "names={names:?}"); + } + + #[test] + fn schema_field_names_are_fixed() { + // Schema must be `row_id` (u64) + `text` (TEXT) regardless of caller's + // field_name argument — matches paimon-java for cross-readability. + let w = + PaimonTantivyWriter::new("ignored_name", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba") + .unwrap(); + let schema = w.index.schema(); + assert!(schema.get_field(PAIMON_ROW_ID_FIELD_NAME).is_ok(), + "schema must have row_id field"); + assert!(schema.get_field(PAIMON_TEXT_FIELD_NAME).is_ok(), + "schema must have text field"); + // Caller-supplied name must NOT appear + assert!(schema.get_field("ignored_name").is_err(), + "caller-supplied field_name must be ignored"); + } + + #[test] + fn archive_uses_big_endian_no_version_header() { + // Strong guard: header must be BE int32 file_count, NOT LE int32 + // version=1 + LE int32 file_count. Any regression to LE/version-header + // would silently break paimon-java cross-read. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello").unwrap(); + let (_, bytes) = w.finish().unwrap(); + let header_be = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + let header_le = i32::from_le_bytes(bytes[0..4].try_into().unwrap()); + // BE file_count is small (single-segment force-merge: ~6-7 files) + assert!(header_be > 0 && header_be < 100, + "expected sensible BE file_count, got BE={header_be} LE={header_le}"); + // LE-decoded header would be a huge number (e.g. 0x06000000), ensuring + // we did NOT regress to the old LE+version layout. + assert_ne!(header_be, header_le, "buffer must be BE-encoded"); + } + + #[test] + fn multi_thread_writer_default() { + // Schema stores row_id explicitly so we no longer enforce + // single-threaded writer. Just verify many docs across threads land + // correctly and force-merge collapses to a single segment. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i, &format!("row {i} apple banana")).unwrap(); + } + let (rows, bytes) = w.finish().unwrap(); + assert_eq!(rows, 200); + assert!(bytes.len() > 4); + // After force-merge there must be exactly one meta.json + segment files. + let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + assert!(file_count >= 2, "force-merged single segment needs ≥ 2 files (meta + segment), got {file_count}"); + } + + #[test] + fn finish_twice_errors() { + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hi").unwrap(); + let _ = w.finish().unwrap(); + let err = w.finish().err().unwrap(); + assert!(err.contains("already finished"), "got: {err}"); + } + + /// Mock collector for FFI streaming tests: push bytes into a Box> + /// pointed to by `ctx`. (No Arc / atomic needed — test is single-threaded.) + extern "C" fn mock_write_collect(ctx: *mut c_void, data: *const u8, len: usize) -> i32 { + let vec = unsafe { &mut *(ctx as *mut Vec) }; + let slice = unsafe { std::slice::from_raw_parts(data, len) }; + vec.extend_from_slice(slice); + 0 + } + + /// Mock that counts the largest single `write` call — sanity check that + /// Rust streams in chunks bounded by the buffer (+ small header fields). + extern "C" fn mock_write_max_chunk( + ctx: *mut c_void, + _data: *const u8, + len: usize, + ) -> i32 { + let max = unsafe { &mut *(ctx as *mut usize) }; + if len > *max { + *max = len; + } + 0 + } + + #[test] + fn ffi_full_path_streaming() { + unsafe { + let field = CString::new("f0").unwrap(); + let mode = CString::new("mix").unwrap(); + let dict = CString::new(dict_dir_from_env().to_str().unwrap()).unwrap(); + let tokenizer = CString::new("paimon_jieba").unwrap(); + let mut handle: *mut PaimonTantivyWriter = std::ptr::null_mut(); + let st = paimon_tantivy_writer_new( + field.as_ptr(), + mode.as_ptr(), + true, + dict.as_ptr(), + tokenizer.as_ptr(), + &mut handle, + ); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert!(!handle.is_null()); + + let txt = "hello world"; + let st = + paimon_tantivy_writer_add(handle, 42u64, txt.as_ptr() as *const c_char, txt.len()); + assert_eq!(st, PaimonTantivyStatus::Ok); + + // Streaming finish: collect bytes into a Vec via FFI callback + let mut out: Vec = Vec::new(); + let cb = PaimonWriteCallbacks { + ctx: &mut out as *mut _ as *mut c_void, + write: mock_write_collect, + }; + let mut rows: i64 = 0; + let st = paimon_tantivy_writer_finish_streaming(handle, cb, &mut rows); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert_eq!(rows, 1); + // BE file_count at byte 0,> 0 + let file_count = i32::from_be_bytes(out[0..4].try_into().unwrap()); + assert!(file_count > 0); + + // double finish must error + let mut out2: Vec = Vec::new(); + let cb2 = PaimonWriteCallbacks { + ctx: &mut out2 as *mut _ as *mut c_void, + write: mock_write_collect, + }; + let mut rows2: i64 = 0; + let st = paimon_tantivy_writer_finish_streaming(handle, cb2, &mut rows2); + assert_eq!(st, PaimonTantivyStatus::InvalidArgument); + + paimon_tantivy_writer_free(handle); + } + } + + #[test] + fn streaming_chunk_size_bounded_by_buffer() { + // After force-merge, a 200-doc index still streams in chunks bounded by + // WRITER_STREAM_BUFFER_SIZE (payload) or small header-field chunks. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i, &format!("row {i} apple banana")).unwrap(); + } + let mut max_chunk: usize = 0; + let cb = PaimonWriteCallbacks { + ctx: &mut max_chunk as *mut _ as *mut c_void, + write: mock_write_max_chunk, + }; + let rows = w.finish_streaming(&cb).unwrap(); + assert_eq!(rows, 200); + assert!( + max_chunk <= WRITER_STREAM_BUFFER_SIZE, + "streaming chunk size {} exceeded buffer {}", + max_chunk, + WRITER_STREAM_BUFFER_SIZE + ); + } + + #[test] + fn streaming_write_callback_error_propagates() { + extern "C" fn always_fail(_ctx: *mut c_void, _data: *const u8, _len: usize) -> i32 { + 7 + } + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello").unwrap(); + let cb = PaimonWriteCallbacks { + ctx: std::ptr::null_mut(), + write: always_fail, + }; + let err = w.finish_streaming(&cb).unwrap_err(); + assert!(err.contains("write callback rc=7"), "got: {err}"); + } + + #[test] + fn ffi_null_writer_invalid() { + unsafe { + let txt = "x"; + let st = paimon_tantivy_writer_add( + std::ptr::null_mut(), + 0u64, + txt.as_ptr() as *const c_char, + txt.len(), + ); + assert_eq!(st, PaimonTantivyStatus::InvalidArgument); + } + } +} diff --git a/docs/code-style.md b/docs/code-style.md index 7cf1d650..23ddb5c6 100644 --- a/docs/code-style.md +++ b/docs/code-style.md @@ -441,4 +441,4 @@ Before submitting a PR, please verify: - [ ] Tests are added or updated for the changed functionality. - [ ] `ASSERT_*` is preferred over `EXPECT_*` in tests. - [ ] No raw `new` / `delete` outside factory methods. -- [ ] PR description follows the [template](.github/PULL_REQUEST_TEMPLATE.md). +- [ ] PR description follows the [template](../.github/PULL_REQUEST_TEMPLATE.md). diff --git a/docs/requirements.txt b/docs/requirements.txt index 818836b6..cf9f23bc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,3 @@ -# -# Note: keep this file in sync with conda_env_sphinx.txt ! -# - breathe myst-parser[linkify] pydata-sphinx-theme~=0.16 diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 28cf190c..60390019 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,7 +1,7 @@ [ { - "name": "0.2.3", - "version": "0.2.3", + "name": "0.3.0", + "version": "0.3.0", "url": "https://paimon.apache.org/docs/cpp/" } ] diff --git a/docs/source/building.rst b/docs/source/building.rst index 8b106c06..8608107a 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -27,13 +27,14 @@ System setup ============ Paimon uses CMake as a build configuration system. We recommend building -out-of-source. For example, you could create ``paimon-cpp/build-release`` -and invoke ``cmake $CMAKE_ARGS ..`` from this directory. +out-of-source. For example, you could create ``paimon-cpp/build`` and invoke +``cmake $CMAKE_ARGS ..`` from this directory. Building requires: * A C++17-enabled compiler. On Linux, gcc 8 and higher should be - sufficient. Windows and MacOS are not supported for now. + sufficient. On macOS, use AppleClang from Xcode Command Line Tools or + LLVM clang from Homebrew. Windows is not supported for now. * At least 2GB of RAM for a minimal build, 8GB for a minimal debug build with tests and 16GB for a full build. @@ -45,6 +46,29 @@ On Ubuntu/Debian you can install the requirements with: build-essential \ cmake +On macOS you can install the requirements with: + +.. code-block:: shell + + xcode-select --install + brew install cmake + +The same CMake build options apply on Linux and macOS. If you prefer upstream +LLVM clang instead of AppleClang on macOS, install LLVM and pass the Homebrew +compiler paths when configuring: + +.. code-block:: shell + + brew install llvm + cmake -B build \ + -DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" \ + -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++" + +When building with upstream Clang on macOS, Paimon uses Apple SDK libc++ +headers by default to avoid incompatibilities in bundled third-party +dependencies. Pass ``-DPAIMON_USE_APPLE_LIBCXX_WITH_CLANG=OFF`` to disable +this behavior. + We also provide a docker template to help you get started quickly. See in ``.devcontainer`` folder for more details. diff --git a/docs/source/conf.py b/docs/source/conf.py index 989ea078..92dfa7c1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -114,7 +114,7 @@ # The master toctree document. master_doc = "index" -version = "0.2.3" +version = "0.3.0" html_theme_options = { "show_toc_level": 2, diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst new file mode 100644 index 00000000..ee17be75 --- /dev/null +++ b/docs/source/examples/benchmark.rst @@ -0,0 +1,88 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================ +Benchmark Usage +================ + +Paimon C++ provides Google Benchmark based cases for append-table write/read and +primary-key table write/MOR read paths. Benchmarks are disabled by default. + +Build +===== + +Enable benchmarks when configuring CMake:: + + cmake -S . -B build -DPAIMON_BUILD_BENCHMARKS=ON + cmake --build build --target paimon-read-write-benchmark + +Run all benchmark cases through CTest:: + + cmake --build build --target benchmark + +Custom Options +============== + +``paimon-read-write-benchmark`` accepts Google Benchmark options plus the Paimon +specific options below: + +``--paimon_source_data_file=`` + Source data file used to build benchmark data. Currently Parquet source files + are supported. + +``--paimon_source_table_path=`` + Read directly from an existing table path for ``BM_Read`` and ``BM_MOR_Read``. + When set, the source loading and pre-write stage are skipped. + +``--paimon_pk_columns=`` + Primary key columns for ``BM_PK_Write`` and ``BM_MOR_Read``. These cases + explicitly use ``bucket=1`` because benchmark batches are written to bucket 0. + +``--paimon_option=:;:`` + Repeatable table options passed through to Paimon. The default table file + format is ``parquet``; use ``--paimon_option file.format:`` to + override it. For ``BM_PK_Write`` and ``BM_MOR_Read``, ``bucket`` is forced to + ``1``. + +Examples +======== + +Append table write:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --benchmark_filter=BM_Write + +Append table read with four prefetch workers:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --benchmark_filter=BM_Read/4 + +Primary-key table write:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --paimon_pk_columns=id \ + --benchmark_filter=BM_PK_Write + +MOR read from an existing table:: + + paimon-read-write-benchmark \ + --paimon_source_table_path /path/table \ + --paimon_pk_columns=id \ + --benchmark_filter=BM_MOR_Read/4 diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index c26b7497..b3114f21 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -23,3 +23,4 @@ Examples write_commit_scan_read clean + benchmark diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index 5b14faac..f0455343 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -27,9 +27,13 @@ User Guide user_guide/schema user_guide/snapshot user_guide/manifest + user_guide/manifest_cache + user_guide/manifest_entry_cache + user_guide/parquet_metadata_cache user_guide/data_types user_guide/primary_key_table user_guide/append_only_table + user_guide/system_tables user_guide/write user_guide/commit user_guide/compaction diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index de794d83..60597dee 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -41,24 +41,29 @@ and `Arrow DataTypes `` where n is the unique name of a field, t is the logical type of a field, d is the description of a field. + + * - ``VARIANT`` + - Struct + - Data type of semi-structured data (e.g. JSON). A variant value contains one of: + a primitive (e.g. integer, string), an array of variant values, or an object + mapping string keys to variant values. + + Variant values are encoded with two binaries following the parquet-format + `Variant Binary Encoding `_ + specification (compatible with the Java Paimon / Spark implementation). In + C++ Paimon, a variant field is represented in an Arrow schema as + ``Struct{value: Binary NOT NULL, metadata: Binary NOT NULL}`` marked with + Paimon-specific field metadata; use ``paimon::Variant::ArrowField`` to + construct such a field, and ``paimon::Variant`` (``FromJson``/``ToJson``/ + ``VariantGet``) to build and inspect values. + + Only the parquet file format supports VARIANT columns. VARIANT cannot be + used as a primary key, partition key or bucket key, and no predicate + pushdown applies to it. + + When writing, variant columns can optionally be *shredded* into typed + parquet columns per the parquet-format + `Variant Shredding `_ + specification by setting ``variant.shreddingSchema`` to a ROW type JSON + whose fields map top-level variant column names to their shredding + types. Alternatively, setting ``variant.inferShreddingSchema`` to + ``true`` infers a shredding schema per file from the first written rows + (tuned by ``variant.shredding.maxSchemaWidth``, which bounds the total + number of shredded fields across all variant columns of the schema, + ``variant.shredding.maxSchemaDepth``, + ``variant.shredding.minFieldCardinalityRatio`` and + ``variant.shredding.maxInferBufferRow``). Inference also covers variant + columns nested inside ROW columns (variants inside arrays or maps stay + unshredded, as in Java Paimon). Readers reassemble shredded columns + transparently. + + When reading, instead of the full variant, specific paths can be + extracted by replacing the variant column in the read schema with a + projection built by ``paimon::VariantAccessBuilder`` (e.g. ``$.a.b`` as + BIGINT). For shredded files only the required typed sub-columns are + read. diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index be585329..bc12c3ae 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -74,8 +74,15 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie - ``MATCH_ALL``: All terms in the query must be present (AND semantics). - ``MATCH_ANY``: Any term in the query can match (OR semantics). - ``PHRASE``: Matches the exact sequence of words (with proximity). -- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). -- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). +- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). The + query is not tokenized. The original prefix is retained, and a pure ASCII alphanumeric prefix + is also matched using the lowercase case-normalization applied to pure ASCII terms at indexing + time. This preserves matches for mixed terms such as ``B超`` while allowing ``THIS`` to match + terms indexed as ``this...``. +- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). The query + is not tokenized, and wildcard operators are preserved. Both the original pattern and an + alternative with each ASCII alphanumeric fragment lowercased are matched, covering pure ASCII + and mixed ASCII/non-ASCII terms. **Special Configuration:** diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst new file mode 100644 index 00000000..0e027509 --- /dev/null +++ b/docs/source/user_guide/manifest_cache.rst @@ -0,0 +1,99 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Manifest Cache +============== + +Overview +-------- + +paimon-cpp caches raw manifest file bytes at the ``ObjectsFile::Read()`` +layer. The cache uses the public ``Cache`` abstraction and is injected through +``ScanContextBuilder`` or ``ReadContextBuilder``. The cache covers data +manifests, manifest lists, and index manifests because they all read through +``ObjectsFile``. + +For repeated ``get``, ``scan``, or batch ``get/scan -f`` requests in the same +process, the same snapshot often reads the same manifest files repeatedly. On a +cache hit, the read path skips remote filesystem ``open/read``, builds an +in-memory input stream from cached bytes, and still runs the format reader, +Arrow decoding, and object deserialization. This design primarily reduces +remote IO latency and bandwidth while keeping cache weight aligned with the +actual cached bytes. + +Configuration +------------- + +Manifest caching is disabled by default. Embedding applications that need it can +provide a custom ``Cache`` implementation and inject it through ``WithCache``. +Manifest reads create cache keys with ``CacheKind::MANIFEST`` internally, so +callers do not need to pass the cache kind through scan or read contexts. The +same cache instance can be reused across multiple scan or read contexts when +process-local sharing is desired. + +Example: + +.. code-block:: cpp + + class RoutingCache : public paimon::Cache { + public: + RoutingCache(std::shared_ptr default_cache, + std::shared_ptr manifest_cache) + : default_cache_(std::move(default_cache)), + manifest_cache_(std::move(manifest_cache)) {} + + paimon::Result> Get( + const std::shared_ptr& key, + std::function>( + const std::shared_ptr&)> supplier) override { + return Select(key)->Get(key, std::move(supplier)); + } + + // Put(), Invalidate(), InvalidateAll(), and Size() route in the same way. + + private: + std::shared_ptr Select( + const std::shared_ptr& key) const { + return key && key->GetKind() == paimon::CacheKind::MANIFEST + ? manifest_cache_ + : default_cache_; + } + + std::shared_ptr default_cache_; + std::shared_ptr manifest_cache_; + }; + + auto cache = std::make_shared( + std::make_shared(), + std::make_shared()); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.WithCache(cache); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.WithCache(cache); + +Passing ``nullptr`` or omitting ``WithCache()`` leaves manifest caching disabled. + +Future Optimizations +-------------------- + +- Add hit, miss, bypass, and eviction metrics to read trace or metrics. +- Add single-flight loading for high-concurrency misses on the same manifest + path. +- Evaluate a decoded-records second-level cache, configurable as a + CPU-vs-memory tradeoff. diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst new file mode 100644 index 00000000..97b33cde --- /dev/null +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -0,0 +1,91 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Manifest Entry Cache +==================== + +Overview +-------- + +Large tables may contain many manifest entries, while a scan may only need a +small subset after bucket, partition, and statistics pruning. The snapshot live +manifest entry cache reduces repeated manifest decoding cost for successive full +scans that target the same bucket. + +The cache stores decoded and merged live manifest entries by table path, branch, +and bucket for ``ScanMode::ALL``. Each cache value can retain several snapshot +results for that bucket. Exact snapshot hits are served from the cache; cache +misses rebuild the target snapshot bucket from the target snapshot's data +manifests and store the rebuilt live entries. + +Request-specific filters are not stored in the cache. Partition, level, and +predicate filters are still evaluated for each scan, so cached entries can be +reused safely across different scan predicates for the same bucket. + +Configuration +------------- + +Manifest entry caching reuses the cache instance provided by +``ScanContextBuilder::WithCache()`` and stores bucket-scoped snapshot entries +under +``CacheKind::SNAPSHOT_LIVE_MANIFEST``: + +.. code-block:: cpp + + auto cache = std::make_shared(128 * 1024 * 1024); + ScanContextBuilder context_builder(table_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + context_builder + .WithCache(cache) + .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .Finish()); + +Cache entries are scoped by table path, branch, and bucket, so they can be +reused across newly created ``TableScan`` and ``FileStoreScan`` instances as +long as they share the same cache object and scan the same bucket. + +``Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS`` controls how many snapshot +results are retained in each table/branch/bucket cache value. Older snapshots in +the same bucket are evicted first. The default value is ``0``, which disables +the cache path. Set it to a positive value to enable the cache when +``ScanContextBuilder::WithCache()`` is also configured. Physical cache eviction +is still controlled by the configured ``Cache`` implementation, for example the +capacity of ``LruCache``. + +If no cache is provided through ``ScanContextBuilder::WithCache()``, this +optimization is skipped. The snapshot manifest entry cache shares the same +``Cache`` interface with raw manifest and data-file footer caches, but it uses a +dedicated ``CacheKind`` and a table/branch/bucket key instead of file byte +ranges. + +Limitations +----------- + +The cache is currently used only for ``ScanMode::ALL`` scans that can determine +a single target bucket. It is skipped for scans without a bucket filter because +reading or deserializing all buckets would be too expensive for selective +queries. It is also skipped for row-range scans because row-range pruning is +applied at manifest-meta level. + +Metrics +------- + +The scan metrics expose existing counters for the last scan: + +- ``lastScannedManifests``: how many manifest files were loaded during this + scan before manifest entry decoding. diff --git a/docs/source/user_guide/parquet_metadata_cache.rst b/docs/source/user_guide/parquet_metadata_cache.rst new file mode 100644 index 00000000..d0e9626e --- /dev/null +++ b/docs/source/user_guide/parquet_metadata_cache.rst @@ -0,0 +1,110 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Parquet Metadata Cache +====================== + +Overview +-------- + +paimon-cpp can cache serialized Parquet metadata footer bytes for Parquet data files. +The cache is used by ``ParquetReaderBuilder`` before opening the Arrow Parquet +reader. On a cache miss, paimon-cpp loads the Parquet file metadata, serializes +it as a complete metadata footer, and stores those bytes in the public +``Cache`` abstraction. On a cache hit, paimon-cpp parses the cached footer bytes into +``parquet::FileMetaData`` and passes the metadata to the Parquet reader. + +The cache stores serialized metadata footer bytes instead of caching a +``parquet::FileMetaData`` instance. This keeps the cache value compact and +similar to manifest cache values: the cache weight follows the actual cached +bytes, while the Parquet library still owns metadata parsing and validation. + +This optimization is useful when the same Parquet files are opened repeatedly +in the same process, for example repeated ``get`` or ``scan`` requests over the +same snapshot. On a cache hit, the read path avoids reading the Parquet footer +bytes from the filesystem again. paimon-cpp still parses the cached footer bytes +into ``parquet::FileMetaData`` for each reader open. Data pages, page indexes, +and column chunks are still read from the file as usual. + +Configuration +------------- + +Parquet metadata caching is disabled by default. Embedding applications that +need it can provide a custom ``Cache`` implementation and inject it through +``ScanContextBuilder`` or ``ReadContextBuilder``. Parquet reader builders +receive the cache from the read context and create cache keys with +``CacheKind::DATA_FILE_FOOTER`` internally. + +The cache key represents the file footer and is created from the file URI with +position ``-1`` and length ``-1``. Callers do not need to construct this key +directly; they only need to route ``CacheKind::DATA_FILE_FOOTER`` entries to an +appropriate cache backend. + +Example: + +.. code-block:: cpp + + class RoutingCache : public paimon::Cache { + public: + RoutingCache(std::shared_ptr default_cache, + std::shared_ptr parquet_metadata_cache) + : default_cache_(std::move(default_cache)), + parquet_metadata_cache_(std::move(parquet_metadata_cache)) {} + + paimon::Result> Get( + const std::shared_ptr& key, + std::function>( + const std::shared_ptr&)> supplier) override { + return Select(key)->Get(key, std::move(supplier)); + } + + // Put(), Invalidate(), InvalidateAll(), and Size() route in the same way. + + private: + std::shared_ptr Select( + const std::shared_ptr& key) const { + return key && key->GetKind() == paimon::CacheKind::DATA_FILE_FOOTER + ? parquet_metadata_cache_ + : default_cache_; + } + + std::shared_ptr default_cache_; + std::shared_ptr parquet_metadata_cache_; + }; + + auto cache = std::make_shared( + std::make_shared(), + std::make_shared()); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.WithCache(cache); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.WithCache(cache); + +Passing ``nullptr`` or omitting ``WithCache()`` leaves Parquet metadata caching +disabled. If a file URI cannot be obtained, paimon-cpp also bypasses the cache +and opens the Parquet file normally. + +Future Optimizations +-------------------- + +- Add hit, miss, bypass, and eviction metrics for Parquet metadata cache. +- Add single-flight loading for high-concurrency misses on the same Parquet + file. +- Evaluate sharing cached metadata footer bytes with page-index prefetch logic when + those read paths can use the same cache abstraction. diff --git a/docs/source/user_guide/prefetch.rst b/docs/source/user_guide/prefetch.rst index 9f62da20..4e601219 100644 --- a/docs/source/user_guide/prefetch.rst +++ b/docs/source/user_guide/prefetch.rst @@ -23,13 +23,13 @@ Prefetch :align: center :width: 100% -In C++ Paimon, we use a multi-producer, single-consumer model to optimize file -reading. The core idea is to split a file into line-based ReadRanges and assign +In Paimon C++, we use a multi-producer, single-consumer model to optimize file +reading. The core idea is to split a file into row-based read ranges and assign them to multiple reader threads (producers). Each reader thread owns an independent result queue that holds its processed RecordBatches. In the main -reader thread (the consumer), we sort the heads of all queues by the ReadRange -start offset in ascending order and select the RecordBatch with the smallest -start offset to ensure globally ordered results. +reader thread (the consumer), we sort the heads of all queues by the read +range's starting row and select the RecordBatch with the smallest starting row +to ensure globally ordered results. Read Range Splitting Strategy ============================= diff --git a/docs/source/user_guide/system_tables.rst b/docs/source/user_guide/system_tables.rst new file mode 100644 index 00000000..6ac01882 --- /dev/null +++ b/docs/source/user_guide/system_tables.rst @@ -0,0 +1,70 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +System Tables +============= + +Paimon C++ supports reading system tables by appending a system table suffix to +the data table path. For example, ``/warehouse/db.db/orders$snapshots`` reads +the snapshots system table for ``orders``. + +Branch-qualified paths are also supported. For example, +``/warehouse/db.db/orders$branch_audit$files`` reads the files system table from +the ``audit`` branch. + +Read-Optimized System Table +--------------------------- + +The read-optimized system table is addressed by the ``$ro`` suffix: + +.. code-block:: text + + /warehouse/db.db/orders$ro + +For primary-key tables, ``$ro`` only plans data files from the highest LSM +level, which is the level produced by full compaction. This avoids merging data +from multiple LSM levels during query planning and reading. The tradeoff is that +the result may lag behind the latest committed data until a full compaction +publishes the newest records into the highest level. + +This stale view is not guaranteed to correspond to any single historical table +snapshot. Full compaction may finish independently for different buckets. When +``$ro`` scans the latest snapshot, the selected highest-level files can +therefore combine bucket states produced by full-compaction commits at different +snapshot IDs. + +For primary-key tables, ``$ro`` also enables value-stats filtering, so file-level +pruning of the reader predicate can be more aggressive than the base table. + +For append-only tables, ``$ro`` has the same read behavior as the base table, +including streaming reads. + +Limitations +~~~~~~~~~~~ + +- Primary-key ``$ro`` scans are batch-only. Streaming scans are not supported. +- Freshness depends on full compaction frequency, and the result may not match + any single historical snapshot across buckets. +- Primary-key tables in bucket-unaware mode are not supported by the current C++ + scan path. + +Typical Usage +~~~~~~~~~~~~~ + +Use ``$ro`` for OLAP or batch workloads that can tolerate stale results and +prefer reading compacted files directly. Use the base table path when the query +must see the latest committed data. diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst index 6fd5c41b..a9d81906 100644 --- a/docs/source/user_guide/write.rst +++ b/docs/source/user_guide/write.rst @@ -128,11 +128,11 @@ produce a correct ``Snapshot``, which commonly includes (but is not limited to): .. note:: The C++ writer supports Append and PK tables and can produce - ``CommitMessage`` objects for both. ``FileStoreCommit`` currently executes - local commits only for append-only tables on non-object-store file systems. - PK and object-store commit messages must be sent to an external control - plane. Changelog is out of scope and should not be emitted in - ``CommitMessage`` until explicitly supported. + ``CommitMessage`` objects for both. ``FileStoreCommit`` supports direct + file-system commits for both table types on non-object-store paths. + Object-store paths require REST catalog commit mode. Changelog is out of + scope and should not be emitted in ``CommitMessage`` until explicitly + supported. Serialization and Deserialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -155,21 +155,23 @@ Operational Flow 1. Writer nodes perform data ingestion and produce Arrow ``RecordBatch`` organized by partition and bucket. -2. Writers flush batches into ORC/Parquet files via registered ``file.format`` - and ``file-system`` backends, producing file-level metadata and per-batch - commit state. +2. Writers flush batches into ORC, Parquet, or Avro files via registered + ``file.format`` and ``file-system`` backends, producing file-level metadata + and per-batch commit state. 3. Each writer invokes ``PrepareCommit``, which: - - Aggregates per-writer state into a ``CommitMessage``. + - Aggregates per-writer state into ``CommitMessage`` objects. - Returns ``CommitMessage`` objects; it does not serialize them. 4. The compute engine gathers ``CommitMessage`` objects from all writers. For cross-process transport, it explicitly calls ``Serialize`` or ``SerializeList`` and carries ``CurrentVersion()`` alongside the payload. -5. For a supported local append-table commit, the engine passes the objects to - ``FileStoreCommit``. For PK tables or object-store paths, it sends the - serialized payload and version to an external control plane. +5. For a direct file-system commit on a non-object-store path, the engine + passes the objects to ``FileStoreCommit`` for either an Append or PK table. + For an object-store path, it enables REST catalog commit mode, calls + ``Commit``, obtains the JSON request from ``GetLastCommitTableRequest``, and + sends that request to the REST catalog. -6. The local committer or external coordinator validates the messages, updates +6. The local committer or REST catalog validates the messages, updates manifests/metadata, and finalizes the snapshot atomically. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 05bdee19..fbaaf1f3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.22) project(example) diff --git a/src/paimon/common/io/cache/cache.h b/include/paimon/cache/cache.h similarity index 55% rename from src/paimon/common/io/cache/cache.h rename to include/paimon/cache/cache.h index d2cfeab1..5bff2a12 100644 --- a/src/paimon/common/io/cache/cache.h +++ b/include/paimon/cache/cache.h @@ -17,25 +17,63 @@ */ #pragma once + +#include #include #include #include #include -#include "paimon/common/io/cache/cache_key.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" +#include "paimon/visibility.h" namespace paimon { class CacheValue; +enum class CacheKind { + DEFAULT, + MANIFEST, + DATA_FILE_FOOTER, + SNAPSHOT_LIVE_MANIFEST, +}; + +class PAIMON_EXPORT CacheKey { + public: + static std::shared_ptr ForPosition(const std::string& file_path, int64_t position, + int32_t length, bool is_index); + static std::shared_ptr ForKind(const std::string& file_path, int64_t position, + int32_t length, CacheKind kind); + static std::shared_ptr ForSnapshotLiveManifestEntries(const std::string& table_path, + const std::string& branch, + int32_t bucket); + + public: + virtual ~CacheKey() = default; + + virtual bool IsIndex() const = 0; + + CacheKind GetKind() const { + return kind_; + } + + virtual bool Equals(const CacheKey& other) const = 0; + + virtual size_t HashCode() const = 0; + + protected: + explicit CacheKey(CacheKind kind) : kind_(kind) {} + + private: + CacheKind kind_ = CacheKind::DEFAULT; +}; -/// Callback invoked when a cache entry is evicted by the LRU policy. using CacheCallback = std::function&)>; class PAIMON_EXPORT Cache { public: virtual ~Cache() = default; + virtual Result> Get( const std::shared_ptr& key, std::function>(const std::shared_ptr&)> @@ -51,31 +89,21 @@ class PAIMON_EXPORT Cache { virtual size_t Size() const = 0; }; -class CacheValue { +class PAIMON_EXPORT CacheValue { public: - CacheValue(const MemorySegment& segment, CacheCallback callback) - : segment_(segment), callback_(std::move(callback)) {} + CacheValue(const MemorySegment& segment, CacheCallback callback); - const MemorySegment& GetSegment() const { - return segment_; - } + ~CacheValue(); - /// Invoke the eviction callback, if one was registered. - void OnEvict(const std::shared_ptr& key) const { - if (callback_) { - callback_(key); - } - } + const MemorySegment& GetSegment() const; - bool operator==(const CacheValue& other) const { - if (this == &other) { - return true; - } - return segment_ == other.segment_; - } + void OnEvict(const std::shared_ptr& key) const; + + bool operator==(const CacheValue& other) const; private: MemorySegment segment_; CacheCallback callback_; }; + } // namespace paimon diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 4b0fd1e7..b5d6bddd 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -171,8 +171,9 @@ class PAIMON_EXPORT Catalog { /// @note This does not check whether the table actually exists. /// /// @param identifier The table identifier containing database and table name. - /// @return A string representing the expected location of the table. - virtual std::string GetTableLocation(const Identifier& identifier) const = 0; + /// @return A result containing the expected location of the table, or an error status on + /// failure. + virtual Result GetTableLocation(const Identifier& identifier) const = 0; /// Returns the root path of the catalog. /// @@ -184,6 +185,11 @@ class PAIMON_EXPORT Catalog { /// @return A shared pointer to the file system instance. virtual std::shared_ptr GetFileSystem() const = 0; + /// Returns the catalog-level options that were passed during catalog creation. + /// + /// @return A const reference to the map of catalog options (key-value pairs). + virtual const std::map& GetOptions() const = 0; + /// Loads the latest schema of a specified table. /// /// @note System tables will not be supported. diff --git a/include/paimon/catalog/identifier.h b/include/paimon/catalog/identifier.h index 62d6ab8f..82ea2e3a 100644 --- a/include/paimon/catalog/identifier.h +++ b/include/paimon/catalog/identifier.h @@ -38,7 +38,7 @@ class PAIMON_EXPORT Identifier { explicit Identifier(const std::string& table); Identifier(const std::string& database, const std::string& table); - bool operator==(const Identifier& other); + bool operator==(const Identifier& other) const; const std::string& GetDatabaseName() const; const std::string& GetTableName() const; Result GetDataTableName() const; @@ -46,7 +46,12 @@ class PAIMON_EXPORT Identifier { Result GetBranchNameOrDefault() const; Result> GetSystemTableName() const; Result IsSystemTable() const; + std::string GetFullName() const; std::string ToString() const; + int32_t HashCode() const; + + public: + static Result FromString(const std::string& full_name); private: Status SplitTableName() const; @@ -60,3 +65,13 @@ class PAIMON_EXPORT Identifier { }; } // namespace paimon + +namespace std { +template <> +struct hash { + size_t operator()(const paimon::Identifier& identifier) const { + return identifier.HashCode(); + } +}; + +} // namespace std diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index b273048f..91acb1c1 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -40,7 +40,7 @@ class PAIMON_EXPORT CommitContext { public: CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& memory_pool, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& options); @@ -62,6 +62,10 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + bool AppendCommitCheckConflict() const { + return append_commit_check_conflict_; + } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -83,6 +87,7 @@ class PAIMON_EXPORT CommitContext { std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; + bool append_commit_check_conflict_; std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; @@ -128,6 +133,11 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); + /// Sets whether append commits should perform conflict checking (default is false). + /// @param append_commit_check_conflict True to enable append conflict checks. + /// @return Reference to this builder for method chaining. + CommitContextBuilder& AppendCommitCheckConflict(bool append_commit_check_conflict); + /// Sets the memory pool to be used for memory allocation during commit operations. /// @param memory_pool Shared pointer to the memory pool instance. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/commit_message.h b/include/paimon/commit_message.h index 45218927..d7cba283 100644 --- a/include/paimon/commit_message.h +++ b/include/paimon/commit_message.h @@ -33,7 +33,7 @@ class CommitMessageSerializer; class MemoryPool; /// Commit message for partition and bucket. Supports serialization and deserialization compatible -/// with the Java version. +/// with Java Paimon. /// /// @note Serialized payloads do not embed their serialization version. Transport /// `CurrentVersion()` alongside the payload and pass it explicitly to `Deserialize()` or @@ -47,7 +47,7 @@ class PAIMON_EXPORT CommitMessage { virtual ~CommitMessage(); /// Serializes a single commit message to a binary string format. - /// The serialized format is compatible with the Java version of Paimon. + /// The serialized format is compatible with Java Paimon. /// The serialization version is not included in the returned payload. /// @param commit_message The commit message to serialize. /// @param pool Memory pool for memory allocation during serialization. diff --git a/include/paimon/data/blob.h b/include/paimon/data/blob.h index ff89714f..0b66ce7b 100644 --- a/include/paimon/data/blob.h +++ b/include/paimon/data/blob.h @@ -100,7 +100,8 @@ class PAIMON_EXPORT Blob { /// @param metadata A map of key-value metadata to be attached to the field. /// @return A result containing a unique pointer to the generated `ArrowSchema` or an error. static Result> ArrowField( - const std::string& field_name, std::unordered_map metadata = {}); + const std::string& field_name, bool nullable = false, + std::unordered_map metadata = {}); private: class Impl; diff --git a/include/paimon/data/shredding/map_shared_shredding_schema_utils.h b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h new file mode 100644 index 00000000..b8470c4d --- /dev/null +++ b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/arrow/abi.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// Parsed file-level meta for one shared-shredding MAP column. +struct PAIMON_EXPORT MapSharedShreddingFieldMeta { + /// field_name -> field_id + std::map name_to_id; + /// field_id -> ordered physical column indices + std::map> field_to_columns; + /// Set of field_ids that ever spilled into __overflow + std::set overflow_field_set; + /// Number of physical columns K in this file + int32_t num_columns = 0; + /// Maximum row width observed in this file + int32_t max_row_width = 0; + + bool operator==(const MapSharedShreddingFieldMeta& other) const { + if (this == &other) { + return true; + } + return name_to_id == other.name_to_id && field_to_columns == other.field_to_columns && + overflow_field_set == other.overflow_field_set && num_columns == other.num_columns && + max_row_width == other.max_row_width; + } +}; + +class PAIMON_EXPORT MapSharedShreddingSchemaUtils { + public: + MapSharedShreddingSchemaUtils() = delete; + ~MapSharedShreddingSchemaUtils() = delete; + + /// Converts a logical schema to a physical schema by replacing shredding MAP columns + /// with their physical Struct representation. + /// @param logical_schema The original Arrow C schema with MAP columns. + /// Ownership of schema resources is transferred to this method. + /// @param field_to_num_columns Map from field name to its physical column count K. + /// Each shredding column can have its own width. + /// @return The exported Arrow C schema for file writing. + static Result> LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema> logical_schema, + const std::map& field_to_num_columns); + + /// Attaches shared-shredding metadata to fields in a physical schema. + /// @param physical_schema The Arrow C physical schema whose fields should receive metadata. + /// Ownership of schema resources is transferred to this method. + /// @param field_name_to_meta Map from physical field name to its shared-shredding metadata. + /// Existing shared-shredding metadata keys on matching fields are overwritten. + /// @param compression Compression codec name for field_dict serialization. + /// @return A new Arrow C schema with shared-shredding metadata attached to matching fields. + static Result> AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema> physical_schema, + const std::map& field_name_to_meta, + const std::string& compression); + + /// Extracts shared-shredding metadata from a named field in a physical schema. + /// @param physical_schema The Arrow C physical schema that contains the target field. + /// Ownership of schema resources is transferred to this method. + /// @param field_name The physical field name whose metadata should be extracted. + /// @return Parsed shared-shredding metadata for the field. + static Result ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name); +}; + +} // namespace paimon diff --git a/include/paimon/data/variant.h b/include/paimon/data/variant.h new file mode 100644 index 00000000..1958d09d --- /dev/null +++ b/include/paimon/data/variant.h @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" +#include "paimon/visibility.h" + +struct ArrowArray; +struct ArrowSchema; + +namespace paimon { + +/// Arguments controlling how a variant value is cast to a target type in `Variant::VariantGet`. +struct PAIMON_EXPORT VariantCastArgs { + /// Whether an invalid cast fails the call (true) or yields SQL NULL (false). + bool fail_on_error = true; + /// The time zone used when rendering TIMESTAMP values. Supported forms are `UTC`/`Z`/`GMT`, + /// fixed offsets such as `+08:00`, and IANA region ids such as `Asia/Shanghai`. + std::string zone_id = "UTC"; +}; + +/// A Variant represents a type that contains one of: 1) Primitive: A type and corresponding +/// value (e.g. INT, STRING); 2) Array: An ordered list of Variant values; 3) Object: An +/// unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not +/// contain duplicate keys. +/// +/// A Variant is encoded with 2 binaries: the value and the metadata, following the parquet +/// Variant Binary Encoding specification (compatible with the Java / Spark implementation). The +/// encoding allows representation of semi-structured data (e.g. JSON) in a form that can be +/// efficiently queried by path. +/// +/// In an Arrow schema, a Variant field is represented as +/// `struct` marked with Paimon-specific field +/// metadata; use `Variant::ArrowField` to construct such a field. +class PAIMON_EXPORT Variant { + public: + ~Variant(); + + /// Parses a JSON string as a Variant (duplicate object keys are rejected). + /// + /// @param json The JSON document text. + /// @param pool The memory pool used for the variant buffers. + /// @return A result containing the created variant or an error. + static Result> FromJson(const std::string& json, + const std::shared_ptr& pool); + + /// Creates a Variant from already-encoded value and metadata binaries (copied into `pool`). + /// + /// @param value The variant value binary. + /// @param value_length The length of the value binary. + /// @param metadata The variant metadata binary. + /// @param metadata_length The length of the metadata binary. + /// @param pool The memory pool used for the variant buffers. + /// @return A result containing the created variant or an error. + static Result> Create(const char* value, uint64_t value_length, + const char* metadata, uint64_t metadata_length, + const std::shared_ptr& pool); + + /// The variant value binary. The view remains valid as long as this variant exists. + std::string_view Value() const; + + /// The variant metadata binary. The view remains valid as long as this variant exists. + std::string_view Metadata() const; + + /// The size of the variant in bytes (value size + metadata size). + int64_t SizeInBytes() const; + + /// Stringifies the variant in JSON format. + /// + /// @param zone_id The time zone used when rendering TIMESTAMP values. + /// @return A result containing the JSON text or an error. + Result ToJson(const std::string& zone_id = "UTC") const; + + /// Extracts a sub-variant value according to a path which starts with a `$`, e.g. `$.key`, + /// `$['key']`, `$["key"]`, `$.array[0]`, and casts the value to the target type. + /// + /// @param path The extraction path. + /// @param target_type The target Arrow type (C data interface, consumed by the call); only + /// scalar types are supported. Use `VariantGetArrow` for nested targets. + /// @param cast_args Cast behavior arguments. + /// @return A result containing the extracted literal, or nullopt for SQL NULL (unmatched + /// path, variant null, or an invalid cast with `fail_on_error == false`). + Result> VariantGet(const std::string& path, + struct ArrowSchema* target_type, + const VariantCastArgs& cast_args) const; + + /// Extracts a sub-variant value according to a path which starts with a `$` and casts the + /// value to the (possibly nested) target field type. + /// + /// In addition to scalar types, the target may be a STRUCT (cast from a variant object, + /// children matched by field name; unmatched children are null), a MAP with string keys + /// (cast from a variant object), a LIST (cast from a variant array), or a variant-marked + /// field created by `Variant::ArrowField` (the sub-variant is deeply re-encoded). + /// + /// @param path The extraction path. + /// @param target_field The target Arrow field (C data interface, consumed by the call). + /// @param cast_args Cast behavior arguments. + /// @return A result containing a length-1 Arrow array of the target type whose single slot + /// holds the result; a null slot represents SQL NULL (unmatched path, variant null, + /// or an invalid cast with `fail_on_error == false`). + Result> VariantGetArrow( + const std::string& path, struct ArrowSchema* target_field, + const VariantCastArgs& cast_args) const; + + /// Extracts a sub-variant value according to a path which starts with a `$` and renders it + /// as JSON text. + /// + /// @param path The extraction path. + /// @param zone_id The time zone used when rendering TIMESTAMP values. + /// @return A result containing the JSON text, or nullopt if the path does not match. + Result> VariantGetJson(const std::string& path, + const std::string& zone_id = "UTC") const; + + /// Creates an Arrow field definition for the Variant type. + /// + /// This function constructs an Arrow Field (internally + /// `struct`) and exports it to the C data + /// interface structure `::ArrowSchema`. It automatically injects Paimon-specific metadata to + /// identify the field as a VARIANT. + /// + /// @param field_name The name of the Arrow field. + /// @param nullable Whether the field is nullable. + /// @param metadata A map of key-value metadata to be attached to the field. + /// @return A result containing a unique pointer to the generated `::ArrowSchema` or an error. + static Result> ArrowField( + const std::string& field_name, bool nullable = true, + std::unordered_map metadata = {}); + + private: + class Impl; + + explicit Variant(std::unique_ptr&& impl); + + std::unique_ptr impl_; +}; + +/// Builds a variant-access projection field: a struct field that replaces a VARIANT column in +/// the read schema so that, instead of the full variant, only the described paths are extracted +/// at read time (reading only the required shredded sub-columns from shredded files). +/// +/// Example: read `$.age` as INT64 and `$.city` as STRING from variant column `v`: +/// +/// VariantAccessBuilder builder; +/// builder.AddField(age_type, "$.age"); +/// builder.AddField(city_type, "$.city"); +/// auto field = builder.Build("v"); // use in ReadContextBuilder::SetReadSchema +/// +/// The resulting struct has one child per added field, named by its position ("0", "1", ...). +class PAIMON_EXPORT VariantAccessBuilder { + public: + VariantAccessBuilder(); + ~VariantAccessBuilder(); + + /// Adds an extracted field. + /// + /// @param target_type The Arrow type the extracted value is cast to (C data interface, + /// consumed by the call). Nested targets follow `Variant::VariantGetArrow` semantics. + /// @param path The extraction path, e.g. `$.a.b` or `$.array[0]`. + /// @param fail_on_error Whether an invalid cast fails the read (true) or yields SQL NULL. + /// @param zone_id The time zone used when rendering TIMESTAMP values. + Status AddField(struct ArrowSchema* target_type, const std::string& path, + bool fail_on_error = true, const std::string& zone_id = "UTC"); + + /// Builds the projection field named `field_name`, to be used in the read schema in place + /// of the variant column with the same name. + Result> Build(const std::string& field_name) const; + + private: + class Impl; + + std::unique_ptr impl_; +}; + +} // namespace paimon diff --git a/include/paimon/defs.h b/include/paimon/defs.h index c38cb492..b938dd38 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -49,6 +49,7 @@ enum class FieldType { MAP = 14, STRUCT = 15, BLOB = 16, + VARIANT = 17, UNKNOWN = 128, }; @@ -119,9 +120,19 @@ struct PAIMON_EXPORT Options { /// append table: the default value is 256 MB. static const char TARGET_FILE_SIZE[]; + /// "target-file-row-num" - Target number of rows per newly written data file. Disabled by + /// default. A file rolls when this or target-file-size is reached, whichever comes first. + /// This limit is enforced at write-batch granularity, so a file may exceed the target by up + /// to one batch. + static const char TARGET_FILE_ROW_NUM[]; + /// "blob.target-file-size" - Target size of a blob file. Default is TARGET_FILE_SIZE. static const char BLOB_TARGET_FILE_SIZE[]; + /// "blob.split-by-file-size" - Whether to consider blob file size as a factor when performing + /// scan splitting. When unset, defaults to the negation of BLOB_AS_DESCRIPTOR. + static const char BLOB_SPLIT_BY_FILE_SIZE[]; + /// "partition.default-name" - The default partition name in case the dynamic partition column /// value is null/empty string. Default is "__DEFAULT_PARTITION__". static const char PARTITION_DEFAULT_NAME[]; @@ -170,6 +181,11 @@ struct PAIMON_EXPORT Options { /// "latest-full", "latest", "from-snapshot", "from-snapshot-full". Default value is "default". static const char SCAN_MODE[]; + /// "scan.manifest-entry-cache.max-snapshots" - Maximum number of snapshot live manifest entry + /// results retained per table, branch, and bucket. Setting it to 0 disables manifest entry + /// cache. Default value is 0. + static const char SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[]; + /// "read.batch-size" - Read batch size for any file format if it supports. /// The default value is 1024. static const char READ_BATCH_SIZE[]; @@ -234,6 +250,16 @@ struct PAIMON_EXPORT Options { /// "commit.max-retries" - Maximum number of retries when commit failed. Default value is 10. static const char COMMIT_MAX_RETRIES[]; + /// "commit.min-retry-wait" - Min retry wait time when commit failed. Default value is 10ms. + static const char COMMIT_MIN_RETRY_WAIT[]; + + /// "commit.max-retry-wait" - Max retry wait time when commit failed. Default value is 10s. + static const char COMMIT_MAX_RETRY_WAIT[]; + + /// "commit.discard-duplicate-files" - Whether to discard duplicate files in commit. + /// Default value is "false". + static const char COMMIT_DISCARD_DUPLICATE_FILES[]; + /// "compaction.max-size-amplification-percent" - The size amplification is defined as the /// amount (in percentage) of additional storage needed to store a single byte of data in the /// merge tree for changelog mode table. Default value is 200. @@ -269,6 +295,15 @@ struct PAIMON_EXPORT Options { /// level 0 files in candidates. Default value is false. static const char COMPACTION_FORCE_UP_LEVEL_0[]; + /// "overwrite-upgrade" - Whether to try upgrading the data files after overwriting a + /// primary key table. Default value is true. + static const char OVERWRITE_UPGRADE[]; + + /// "dynamic-partition-overwrite" - Whether only overwrite dynamic partition when + /// overwriting a partitioned table with dynamic partition columns. Works only when + /// the table has partition keys. Default value is true. + static const char DYNAMIC_PARTITION_OVERWRITE[]; + /// "lookup-compact.max-interval" - The max interval for a gentle mode lookup compaction to be /// triggered. For every interval, a forced lookup compaction will be performed to flush L0 /// files to higher level. This option is only valid when lookup-compact mode is gentle. No @@ -294,6 +329,15 @@ struct PAIMON_EXPORT Options { /// "ignore-delete" - Whether to ignore delete records. Default value is "false". static const char IGNORE_DELETE[]; + /// "first-row.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`. + static const char FALLBACK_FIRST_ROW_IGNORE_DELETE[]; + + /// "deduplicate.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`. + static const char FALLBACK_DEDUPLICATE_IGNORE_DELETE[]; + + /// "partial-update.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`. + static const char FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[]; + /// "fields.default-aggregate-function" - Default aggregate function of all fields for /// partial-update and aggregate merge function. static const char FIELDS_DEFAULT_AGG_FUNC[]; @@ -369,7 +413,58 @@ struct PAIMON_EXPORT Options { /// "partition.legacy-name" - The legacy partition name is using `ToString` for all types. If /// false, using casting to string for all types. Default value is "true". static const char PARTITION_GENERATE_LEGACY_NAME[]; - /// "blob-as-descriptor" - Read and write blob field using blob descriptor rather than blob + /// "map.storage-layout" - Suffix for per-column MAP storage layout configuration. + /// Used as `fields..map.storage-layout`. Values: "default" (standard KV arrays) + /// or "shared-shredding" (columnar shredding with column reuse). Default is "default". + /// If set "shared-shredding", the column must be of type MAP. Each column must be + /// configured individually. For example, to enable shared-shredding layout for two columns + /// "metrics" and "tags": + /// fields.metrics.map.storage-layout = shared-shredding + /// fields.tags.map.storage-layout = shared-shredding + static const char MAP_STORAGE_LAYOUT[]; + /// "map.shared-shredding.max-columns" - Suffix for per-column upper bound K_max configuration. + /// Used as `fields..map.shared-shredding.max-columns`. Only effective when + /// map.storage-layout = shared-shredding. Rows with more fields than K_max spill to + /// __overflow. Default value is 256. Each column can have its own max-columns setting. + static const char MAP_SHARED_SHREDDING_MAX_COLUMNS[]; + /// "map.shared-shredding.column-placement-policy" - Suffix for per-column shared-shredding + /// physical column placement policy. + /// Used as `fields..map.shared-shredding.column-placement-policy`. + /// Values: "plain", "sequential" and "lru". Default value is "lru". + /// Only effective when map.storage-layout = shared-shredding. + static const char MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[]; + + /// "variant.shreddingSchema" - The Variant shredding schema for writing: a ROW type JSON + /// whose fields map variant column names to their shredding types. No default value. + static const char VARIANT_SHREDDING_SCHEMA[]; + /// "parquet.variant.shreddingSchema" - Fallback key of "variant.shreddingSchema". + static const char PARQUET_VARIANT_SHREDDING_SCHEMA[]; + /// "variant.inferShreddingSchema" - Whether to automatically infer the shredding schema when + /// writing Variant columns. Default value is "false". + static const char VARIANT_INFER_SHREDDING_SCHEMA[]; + /// "variant.shredding.inferenceMode" - "per-file" or "adaptive". Default is "per-file". + static const char VARIANT_SHREDDING_INFERENCE_MODE[]; + /// "variant.shredding.maxSchemaWidth" - Maximum number of shredded fields allowed in an + /// inferred schema. Default value is 300. + static const char VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[]; + /// "variant.shredding.maxSchemaDepth" - Maximum traversal depth in Variant values during + /// schema inference. Default value is 50. + static const char VARIANT_SHREDDING_MAX_SCHEMA_DEPTH[]; + /// "variant.shredding.minFieldCardinalityRatio" - Minimum fraction of rows that must contain + /// a field for it to be shredded. Fields below this threshold stay in the un-shredded + /// Variant binary. Default value is 0.1. + static const char VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO[]; + /// "variant.shredding.maxInferBufferRow" - Maximum number of rows to buffer for schema + /// inference. Default value is 4096. + static const char VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[]; + /// "variant.shredding.adaptive.maxInferBufferRow" - Maximum prefix rows sampled after the + /// first file in an adaptive session. Default value is 256. + static const char VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW[]; + /// "variant.shredding.adaptive.retentionRatio" - Minimum combined ratio for retaining a + /// previously selected path. Default value is 0.05. + static const char VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO[]; + + /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob /// bytes. Default value is "false". static const char BLOB_AS_DESCRIPTOR[]; /// "blob-field" - Specifies column names that should be stored as blob type. This is used @@ -386,15 +481,24 @@ struct PAIMON_EXPORT Options { /// serialized BlobViewStruct bytes inline in data files and resolve from upstream tables at /// read time. No default value. static const char BLOB_VIEW_FIELD[]; - /// "blob-external-storage-field" - Comma-separated BLOB field names (must be a subset of - /// blob-descriptor-field ) whose raw data will be written to external storage at write time. - /// The external storage path is configured via blob-external-storage-path. Orphan file cleanup - /// is not applied to that path. No default value. - static const char BLOB_EXTERNAL_STORAGE_FIELD[]; - /// "blob-external-storage-path" - The external storage path where raw BLOB data from fields - /// configured by 'blob-external-storage-field' is written at write time. Orphan file cleanup is - /// not applied to this path. No default value. - static const char BLOB_EXTERNAL_STORAGE_PATH[]; + /// "blob-view.resolve.enabled" - Whether to resolve blob-view-field values from upstream + /// tables at read time. Set to false to preserve serialized BlobViewStruct bytes when + /// forwarding blob view values to another blob-view table. Default value is "true". + static const char BLOB_VIEW_RESOLVE_ENABLED[]; + /// "blob-view-upstream-warehouse" - Since the catalog capabilities are partially missing, when + /// Blob View is enabled, cpp paimon cannot automatically obtain the upstream table warehouse + /// path and requires manual configuration by the user. No default value. + static const char BLOB_VIEW_UPSTREAM_WAREHOUSE[]; + /// "blob-write-null-on-missing-file" - Whether to write NULL for a descriptor BLOB value when + /// the referenced file does not exist at write time. When false, a missing file is treated + /// like any other fetch failure, following "blob-write-null-on-fetch-failure". Default value + /// is "false". + static const char BLOB_WRITE_NULL_ON_MISSING_FILE[]; + /// "blob-write-null-on-fetch-failure" - Whether to write NULL for a descriptor BLOB value when + /// the referenced data cannot be fetched at write time (e.g. invalid descriptor or invalid + /// offset). A missing file is handled by "blob-write-null-on-missing-file" when that option is + /// enabled. When false, the write fails when the descriptor is read. Default value is "false". + static const char BLOB_WRITE_NULL_ON_FETCH_FAILURE[]; /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true". static const char GLOBAL_INDEX_ENABLED[]; /// "global-index.thread-num" - The maximum number of concurrent scanner for global index. No @@ -432,6 +536,12 @@ struct PAIMON_EXPORT Options { /// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This /// option is used along with dedicated compact jobs. Default value is "false". static const char WRITE_ONLY[]; + /// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This + /// option is used by commit conflict checks. Default value is "false". + static const char BUCKET_APPEND_ORDERED[]; + /// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for + /// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan". + static const char WRITE_SEQUENCE_NUMBER_INIT_MODE[]; /// "compaction.min.file-num" - For file set [f_0,...,f_N], the minimum file number to trigger a /// compaction for append-only table. Default value is 5. static const char COMPACTION_MIN_FILE_NUM[]; diff --git a/include/paimon/executor.h b/include/paimon/executor.h index d4203e9e..7092af7b 100644 --- a/include/paimon/executor.h +++ b/include/paimon/executor.h @@ -23,6 +23,7 @@ #include #include +#include "paimon/result.h" #include "paimon/visibility.h" namespace paimon { @@ -37,7 +38,7 @@ PAIMON_EXPORT std::shared_ptr GetGlobalDefaultExecutor(); PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(); /// Create a default implementation of executor with specified thread_count. -PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(uint32_t thread_count); +PAIMON_EXPORT Result> CreateDefaultExecutor(uint32_t thread_count); /// Interface class for defining basic operations of a task executor. /// @@ -57,6 +58,9 @@ class PAIMON_EXPORT Executor { /// Shutdown the executor immediately, discarding all pending tasks. virtual void ShutdownNow() = 0; + + /// Get thread number. + virtual uint32_t GetThreadNum() const = 0; }; } // namespace paimon diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 66d3d390..82bbc003 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -44,10 +44,10 @@ class CommitMessage; /// The `FileStoreCommit` class provides interfaces for committing changes, expiring old snapshots, /// dropping partitions, and retrieving commit metrics. /// -/// @note Local commit execution currently supports append-only tables on non-object-store file -/// systems. `Create()` returns `NotImplemented` for primary-key tables and object-store paths. -/// Primary-key writers can still produce `CommitMessage` objects; those messages must be committed -/// by an external control plane. +/// @note Direct file-system commits support append-only and primary-key tables on non-object-store +/// paths. Object-store paths require REST catalog commit mode: enable it with +/// `CommitContextBuilder::UseRESTCatalogCommit()`, call `Commit()` or `FilterAndCommit()`, and then +/// retrieve the request with `GetLastCommitTableRequest()`. class PAIMON_EXPORT FileStoreCommit { public: /// Create an instance of `FileStoreCommit`. @@ -91,7 +91,7 @@ class PAIMON_EXPORT FileStoreCommit { /// Overwrite from manifest committable and partition. /// - /// @param partitions A single partition maps each partition key to a partition value. Depending + /// @param partition A single partition maps each partition key to a partition value. Depending /// on the user-defined statement, the partition might not include all partition keys. Also /// note that this partition does not necessarily equal to the partitions of the newly added /// key-values. This is just the partition to be cleaned up. @@ -100,7 +100,7 @@ class PAIMON_EXPORT FileStoreCommit { /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. - virtual Status Overwrite(const std::vector>& partitions, + virtual Status Overwrite(const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; @@ -108,14 +108,14 @@ class PAIMON_EXPORT FileStoreCommit { /// This is a temporary interface for internal use. It will be removed in a future version. /// Please do not rely on it for long-term use. /// - /// @param partitions Description of the partitions. + /// @param partition Description of the partition. /// @param commit_messages Description of the commit messages. /// @param commit_identifier Unique identifier. /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. virtual Result FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; @@ -143,6 +143,41 @@ class PAIMON_EXPORT FileStoreCommit { virtual Status DropPartition(const std::vector>& partitions, int64_t commit_identifier) = 0; + /// Truncate the whole table by overwriting all partitions with empty data. The generated + /// snapshot has commit kind OVERWRITE. + /// + /// @param commit_identifier An identifier for the commit operation. + /// @return Status indicating the success or failure of the truncate operation. + virtual Status TruncateTable(int64_t commit_identifier) = 0; + + /// Abort an unsuccessful commit. The data and index files described by the given commit + /// messages will be deleted on a best-effort basis (delete failures are ignored). + /// + /// @param commit_messages A vector of commit messages whose files should be cleaned up. + /// @return Status indicating the success or failure of the abort operation. + virtual Status Abort(const std::vector>& commit_messages) = 0; + + /// Roll back to the target snapshot and materialize it as the latest snapshot. + /// + /// Reads the surviving files of both the current latest snapshot and the target + /// snapshot, then commits an OVERWRITE snapshot whose visible state equals the target. + /// + /// @param target_snapshot_id The snapshot id to roll back to. + /// @return Result; true if the atomic commit succeeded. Returns an error status if + /// there is no latest snapshot or the target snapshot does not exist. + virtual Result RollbackToAsLatest(int64_t target_snapshot_id) = 0; + + /// Configure row-id conflict checking from a specific snapshot id. + /// + /// If set to a snapshot id, commit conflict detection will additionally validate row-id + /// conflicts against snapshots after that id. Passing std::nullopt disables this behavior. + /// + /// @param row_id_check_from_snapshot Snapshot id to start row-id conflict checks from, or + /// std::nullopt to disable. + /// @return Current commit object for chaining. + virtual FileStoreCommit& RowIdCheckConflict( + std::optional row_id_check_from_snapshot) = 0; + /// Retrieve metrics related to commit operations. /// /// @return A shared pointer to a `Metrics` object containing commit metrics. diff --git a/include/paimon/format/format_writer.h b/include/paimon/format/format_writer.h index cf570b3e..854a0a9b 100644 --- a/include/paimon/format/format_writer.h +++ b/include/paimon/format/format_writer.h @@ -18,8 +18,11 @@ #pragma once +#include #include +#include +#include "paimon/status.h" #include "paimon/type_fwd.h" struct ArrowArray; @@ -68,6 +71,10 @@ class PAIMON_EXPORT FormatWriter { /// Get metrics of the writer /// @return The accumulated writer metrics to current state. virtual std::shared_ptr GetWriterMetrics() const = 0; + + /// Adds metadata to the file footer. Values are encoded by each format writer + /// before being persisted. Must be called before Finish(). + virtual Status AddMetadata(const std::map& metadata) = 0; }; } // namespace paimon diff --git a/include/paimon/format/reader_builder.h b/include/paimon/format/reader_builder.h index ca2021c3..b0837a26 100644 --- a/include/paimon/format/reader_builder.h +++ b/include/paimon/format/reader_builder.h @@ -25,8 +25,9 @@ #include "paimon/type_fwd.h" namespace paimon { +class Cache; -/// Create a file batch reader based on the file path. Allows you to specify memory pool. +/// Create a file batch reader based on an input stream. Allows you to specify memory pool. class PAIMON_EXPORT ReaderBuilder { public: virtual ~ReaderBuilder() = default; @@ -34,6 +35,12 @@ class PAIMON_EXPORT ReaderBuilder { /// Set memory pool to use. virtual ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) = 0; + /// Inject a cache for reader-specific immutable metadata. + virtual ReaderBuilder* WithCache(const std::shared_ptr& cache) { + (void)cache; + return this; + } + /// Build a file batch reader based on the created `InputStream`. virtual Result> Build( const std::shared_ptr& path) const = 0; diff --git a/include/paimon/fs/file_system.h b/include/paimon/fs/file_system.h index f2388dc3..bee3ecc8 100644 --- a/include/paimon/fs/file_system.h +++ b/include/paimon/fs/file_system.h @@ -75,7 +75,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @return Result containing the actual number of bytes read on success, or an error status on /// failure. /// @note The stream position advances by the number of bytes actually read. - virtual Result Read(char* buffer, uint32_t size) = 0; + virtual Result Read(char* buffer, int64_t size) = 0; /// Read data from given position in the stream. /// @@ -85,7 +85,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @param[out] buffer The buffer to store the read content. /// @param size The number of bytes to read. /// @param offset The position in the stream to read from. - virtual Result Read(char* buffer, uint32_t size, uint64_t offset) = 0; + virtual Result Read(char* buffer, int64_t size, int64_t offset) = 0; /// Asynchronously read data from the input stream. /// @@ -100,7 +100,8 @@ class PAIMON_EXPORT InputStream : public Stream { /// @param callback The callback function to be invoked upon completion of the read operation. /// The callback will receive a Status object indicating the success or failure /// of the read operation. - virtual void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + /// @note The caller must keep the input stream and buffer alive until the callback is invoked. + virtual void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) = 0; /// Get an identifier that uniquely identify the underlying content. @@ -109,7 +110,7 @@ class PAIMON_EXPORT InputStream : public Stream { virtual Result GetUri() const = 0; /// Get the total length of the file in bytes. - virtual Result Length() const = 0; + virtual Result Length() const = 0; }; /// Abstract class for output stream operations. @@ -123,7 +124,7 @@ class PAIMON_EXPORT OutputStream : public Stream { /// @return Result containing the actual number of bytes written on success, or an error status /// on failure. /// @note The stream position advances by the number of bytes actually written. - virtual Result Write(const char* buffer, uint32_t size) = 0; + virtual Result Write(const char* buffer, int64_t size) = 0; /// Flush pending data to the disk. virtual Status Flush() = 0; @@ -162,7 +163,7 @@ class PAIMON_EXPORT FileStatus { /// Get the size of the file in bytes. /// @note For directories, this method is undefined behavior. - virtual uint64_t GetLen() const = 0; + virtual int64_t GetLen() const = 0; /// Check if this entry represents a directory. virtual bool IsDir() const = 0; diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index f4497b7e..10ace821 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -19,11 +19,11 @@ #pragma once +#include #include #include #include "paimon/memory/bytes.h" -#include "paimon/utils/range.h" namespace paimon { /// Metadata describing a single file entry in a global index. diff --git a/include/paimon/global_index/global_index_reader.h b/include/paimon/global_index/global_index_reader.h index 36bb201a..ca3f4c9b 100644 --- a/include/paimon/global_index/global_index_reader.h +++ b/include/paimon/global_index/global_index_reader.h @@ -50,7 +50,7 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor +#include #include #include @@ -39,6 +40,9 @@ class PAIMON_EXPORT GlobalIndexer { public: virtual ~GlobalIndexer() = default; + /// Returns additional table fields required during index construction. + virtual Result>> GetExtraFieldNames() const = 0; + /// Creates a writer for building a global index on a specific field. /// /// @param field_name Name of the field to be indexed. diff --git a/include/paimon/global_index/global_indexer_factory.h b/include/paimon/global_index/global_indexer_factory.h index e3d3069c..1181878d 100644 --- a/include/paimon/global_index/global_indexer_factory.h +++ b/include/paimon/global_index/global_indexer_factory.h @@ -35,7 +35,7 @@ class PAIMON_EXPORT GlobalIndexerFactory : public Factory { public: ~GlobalIndexerFactory() override = default; - /// Suffix used to distinguish global index identifiers (e.g., "bitmap-global"). + /// Suffix used to distinguish global index identifiers (e.g., "btree-global"). static const char GLOBAL_INDEX_IDENTIFIER_SUFFIX[]; /// Creates a `GlobalIndexer` instance by looking up a registered factory using an identifier. @@ -44,11 +44,11 @@ class PAIMON_EXPORT GlobalIndexerFactory : public Factory { /// (e.g., "-global") to form the full key used for factory lookup. This ensures namespace /// separation between file and global index types. /// - /// @param identifier The base name of the index type (e.g., "bitmap"). + /// @param identifier The base name of the index type (e.g., "btree"). /// @param options Configuration parameters for the indexer. /// @return A `Result` containing a unique pointer to the created `GlobalIndexer`, /// or an error if creation fails. - /// @return nullptr if no matching factory. + /// @return nullptr if no matching factory exists or the index type is temporarily unsupported. static Result> Get( const std::string& identifier, const std::map& options); diff --git a/include/paimon/io/buffered_input_stream.h b/include/paimon/io/buffered_input_stream.h index 2af6b731..6b172cb9 100644 --- a/include/paimon/io/buffered_input_stream.h +++ b/include/paimon/io/buffered_input_stream.h @@ -45,7 +45,7 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { /// @param in The underlying input stream to wrap. /// @param buffer_size Size of the internal buffer in bytes. /// @param pool Memory pool for buffer allocation. - BufferedInputStream(const std::shared_ptr& in, int32_t buffer_size, + BufferedInputStream(const std::shared_ptr& in, int64_t buffer_size, MemoryPool* pool); ~BufferedInputStream() noexcept override; @@ -54,20 +54,20 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override; + Result Length() const override; Status Close() override; Result GetUri() const override; - static constexpr int32_t DEFAULT_BUFFER_SIZE = 8192; + static constexpr int64_t DEFAULT_BUFFER_SIZE = 8192; private: /// Fill the internal buffer from the underlying stream. @@ -75,15 +75,15 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { /// Internal read implementation. /// @pre size > 0 - Result InnerRead(char* buffer, int32_t size); + Result InnerRead(char* buffer, int64_t size); /// Validate that the expected number of bytes were read. - Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const; + Status AssertReadLength(int64_t read_length, int64_t actual_read_length) const; private: - int32_t buffer_size_; - int32_t pos_ = 0; - int32_t count_ = 0; + int64_t buffer_size_; + int64_t pos_ = 0; + int64_t count_ = 0; std::unique_ptr buffer_; std::shared_ptr in_; }; diff --git a/include/paimon/io/byte_array_input_stream.h b/include/paimon/io/byte_array_input_stream.h index e66c5e8a..ab2d2f19 100644 --- a/include/paimon/io/byte_array_input_stream.h +++ b/include/paimon/io/byte_array_input_stream.h @@ -32,7 +32,7 @@ namespace paimon { /// Input stream for memory buffer, inherits from `InputStream`. class PAIMON_EXPORT ByteArrayInputStream : public InputStream { public: - ByteArrayInputStream(const char* buffer, uint64_t length); + ByteArrayInputStream(const char* buffer, int64_t length); ~ByteArrayInputStream() override = default; /// @return The raw data pointer of current pos. @@ -44,14 +44,14 @@ class PAIMON_EXPORT ByteArrayInputStream : public InputStream { return position_; } - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override { + Result Length() const override { return length_; } @@ -61,7 +61,7 @@ class PAIMON_EXPORT ByteArrayInputStream : public InputStream { private: const char* buffer_; - const uint64_t length_; + const int64_t length_; int64_t position_; }; } // namespace paimon diff --git a/include/paimon/io/data_input_stream.h b/include/paimon/io/data_input_stream.h index 8dc15a2c..d954003e 100644 --- a/include/paimon/io/data_input_stream.h +++ b/include/paimon/io/data_input_stream.h @@ -58,7 +58,7 @@ class PAIMON_EXPORT DataInputStream { /// Read raw data of specified size from the stream. /// @param data Buffer to store the read data. /// @param size Number of bytes to read. - Status Read(char* data, uint32_t size) const; + Status Read(char* data, int64_t size) const; /// Read string from the stream. /// @note First read length (int16), then read string bytes. @@ -68,7 +68,7 @@ class PAIMON_EXPORT DataInputStream { Result GetPos() const; /// Get the total length of the underlying input stream. - Result Length() const; + Result Length() const; /// Set the byte order for endianness conversion. /// @param order The byte order to use `PAIMON_BIG_ENDIAN` or `PAIMON_LITTLE_ENDIAN`. @@ -80,11 +80,11 @@ class PAIMON_EXPORT DataInputStream { /// Validate that the expected number of bytes were read. /// @param read_length Expected number of bytes to read. /// @param actual_read_length Actual number of bytes read. - Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const; + Status AssertReadLength(int64_t read_length, int64_t actual_read_length) const; /// Check if there are enough bytes available to read. /// @param need_length Number of bytes needed. - Status AssertBoundary(int32_t need_length) const; + Status AssertBoundary(int64_t need_length) const; /// Determine if byte swapping is needed based on current byte order and system endianness. /// @return `true` if byte swapping is required, `false` otherwise. diff --git a/include/paimon/memory/bytes.h b/include/paimon/memory/bytes.h index b6f5280f..5c7fb716 100644 --- a/include/paimon/memory/bytes.h +++ b/include/paimon/memory/bytes.h @@ -81,7 +81,7 @@ class PAIMON_EXPORT Bytes { /// @param length Number of bytes to allocate. /// @param pool Memory pool to use for allocation. /// @return Unique pointer to the newly allocated Bytes object. - static PAIMON_UNIQUE_PTR AllocateBytes(int32_t length, MemoryPool* pool); + static PAIMON_UNIQUE_PTR AllocateBytes(size_t length, MemoryPool* pool); /// Allocate a new Bytes object from string data. /// diff --git a/include/paimon/memory/memory_pool.h b/include/paimon/memory/memory_pool.h index b4ed71c7..54176dff 100644 --- a/include/paimon/memory/memory_pool.h +++ b/include/paimon/memory/memory_pool.h @@ -58,7 +58,8 @@ class PAIMON_EXPORT MemoryPool { /// /// @param size Number of bytes to allocate. /// @param alignment Memory alignment requirement (0 for default alignment). - /// @return Pointer to allocated memory, or nullptr on failure. + /// @return Pointer to allocated memory. + /// @throws std::bad_alloc if the allocation fails. virtual void* Malloc(uint64_t size, uint64_t alignment = 0) = 0; /// Reallocate memory to a new size. diff --git a/src/paimon/common/memory/memory_segment.h b/include/paimon/memory/memory_segment.h similarity index 90% rename from src/paimon/common/memory/memory_segment.h rename to include/paimon/memory/memory_segment.h index a98e5d8b..f40526ae 100644 --- a/src/paimon/common/memory/memory_segment.h +++ b/include/paimon/memory/memory_segment.h @@ -25,7 +25,6 @@ #include #include -#include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/visibility.h" @@ -112,14 +111,18 @@ class PAIMON_EXPORT MemorySegment { inline void Get(int32_t index, T* dst, int32_t offset, int32_t length) const { assert(static_cast(dst->size()) >= (offset + length)); assert(size_ >= (index + length)); - std::memcpy(const_cast(dst->data()) + offset, data_ + index, length); + if (length != 0) { + std::memcpy(const_cast(dst->data()) + offset, data_ + index, length); + } } template inline void Put(int32_t index, const T& src, int32_t offset, int32_t length) { assert(static_cast(src.size()) >= (offset + length)); assert(size_ >= (index + length)); - std::memcpy(MutableData() + index, src.data() + offset, length); + if (length != 0) { + std::memcpy(MutableData() + index, src.data() + offset, length); + } } template @@ -137,26 +140,23 @@ class PAIMON_EXPORT MemorySegment { std::memcpy(MutableData() + index, &value, sizeof(T)); } - inline uint64_t GetLongBigEndian(int32_t index) const { - auto value = GetValue(index); - if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) { - return EndianSwapValue(value); - } - return value; - } + uint64_t GetLongBigEndian(int32_t index) const; void CopyTo(int32_t offset, MemorySegment* target, int32_t target_offset, int32_t num_bytes) const { assert(offset >= 0); assert(target_offset >= 0); assert(num_bytes >= 0); - - std::memcpy(target->MutableData() + target_offset, data_ + offset, num_bytes); + if (num_bytes != 0) { + std::memcpy(target->MutableData() + target_offset, data_ + offset, num_bytes); + } } void CopyToUnsafe(int32_t offset, void* target, int32_t target_offset, int32_t num_bytes) const { - std::memcpy(static_cast(target) + target_offset, data_ + offset, num_bytes); + if (num_bytes != 0) { + std::memcpy(static_cast(target) + target_offset, data_ + offset, num_bytes); + } } int32_t Compare(const MemorySegment& seg2, int32_t offset1, int32_t offset2, int32_t len) const; diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h index bd6ba1ea..3e0676c5 100644 --- a/include/paimon/predicate/full_text_search.h +++ b/include/paimon/predicate/full_text_search.h @@ -48,22 +48,27 @@ struct PAIMON_EXPORT FullTextSearch { FullTextSearch(const std::string& _field_name, std::optional _limit, const std::string& _query, const SearchType& _search_type, - const std::optional& _pre_filter) + const std::optional& _pre_filter, bool _with_score = false, + std::optional _min_score = std::nullopt) : field_name(_field_name), limit(_limit), query(_query), search_type(_search_type), - pre_filter(_pre_filter) {} + pre_filter(_pre_filter), + with_score(_with_score), + min_score(_min_score) {} std::shared_ptr ReplacePreFilter( const std::optional& _pre_filter) const { - return std::make_shared(field_name, limit, query, search_type, _pre_filter); + return std::make_shared(field_name, limit, query, search_type, _pre_filter, + with_score, min_score); } /// Name of the field to search within (must be a full-text indexed field). std::string field_name; - /// Maximum number of documents to return. If set, limit ordered by top scores. Otherwise, no - /// score return. + /// Maximum number of documents to return. Purely a truncation switch, + /// orthogonal to `with_score`: set `with_score = true` to get relevance + /// scores; a non-empty `limit` does not by itself imply scoring. std::optional limit; /// The query string to search for. The interpretation depends on search_type: /// @@ -74,10 +79,14 @@ struct PAIMON_EXPORT FullTextSearch { /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed. /// /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner). - /// Only the prefix part is considered; analysis will not be applied. + /// The query is not tokenized or filtered for stop words. The original prefix is retained, + /// and a prefix consisting entirely of ASCII letters and digits is also matched using the + /// lowercase case-normalization applied to pure ASCII terms at indexing time. /// /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e"). - /// Not passed through analyzer — matched directly against indexed terms. + /// The query is not tokenized or filtered for stop words. The wildcard operators are + /// preserved. Both the original pattern and an alternative with each ASCII alphanumeric + /// fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms. /// /// @note Analyzer consistency between indexing and querying is critical for correctness. std::string query; @@ -87,5 +96,26 @@ struct PAIMON_EXPORT FullTextSearch { /// Only rows whose global row ID is present in `pre_filter` will be included during search. /// If not set, all rows will be included. std::optional pre_filter; + /// Whether to compute and return relevance scores (e.g. BM25). The 4-path matrix: + /// - `with_score=false, limit=nullopt` → BitmapGlobalIndexResult (all rows, no score) + /// - `with_score=false, limit=N` → BitmapGlobalIndexResult (any N matches, unscored) + /// - `with_score=true, limit=nullopt` → BitmapScoredGlobalIndexResult (all rows + all scores) + /// - `with_score=true, limit=N` → BitmapScoredGlobalIndexResult (top-N by score + + /// scores) + /// + /// For plain `LIMIT N` without ORDER BY (the common case when an online + /// engine, e.g. StarRocks, pushes down a predicate) set `with_score=false, + /// limit=N` — the unscored fast path. For top-N by relevance use + /// `with_score=true, limit=N` and drop the scores in the caller if unneeded. + /// + /// Default is `false` to avoid score computation overhead for callers that don't need it. + bool with_score = false; + /// Minimum relevance-score threshold (exclusive); results with score ≤ this value are + /// excluded. The score is whatever the backend's similarity produces (e.g. BM25 for + /// tantivy, classic TF-IDF for lucene), so a threshold is not directly comparable across + /// backends. Only meaningful when scoring is active (`with_score = true` or `limit` set); + /// applied before truncation so low-score documents never occupy limit slots. + /// Default is nullopt (no threshold filtering). + std::optional min_score; }; } // namespace paimon diff --git a/include/paimon/predicate/predicate.h b/include/paimon/predicate/predicate.h index bd2dbb49..6c094eee 100644 --- a/include/paimon/predicate/predicate.h +++ b/include/paimon/predicate/predicate.h @@ -33,7 +33,7 @@ class Function; /// @see PredicateBuilder class PAIMON_EXPORT Predicate { public: - virtual ~Predicate() = default; + virtual ~Predicate(); virtual bool operator==(const Predicate& other) const = 0; virtual const Function& GetFunction() const = 0; diff --git a/include/paimon/predicate/vector_search.h b/include/paimon/predicate/vector_search.h index b77abd23..44852162 100644 --- a/include/paimon/predicate/vector_search.h +++ b/include/paimon/predicate/vector_search.h @@ -76,6 +76,10 @@ struct PAIMON_EXPORT VectorSearch { /// context-aware filtering at query time. /// @note All fields referenced in the predicate must have been materialized /// in the index during build to ensure availability. + /// @note For tag-based vector indexes, fields referenced by the predicate + /// must keep the same names and types as the fields used during index + /// construction. Indexes built with tag fields must not be reused across + /// schema evolution that renames or changes those tag fields. std::shared_ptr predicate; /// The distance metric to use for this query, if explicitly specified. /// If set, this value must match the distance type used by the index (e.g., EUCLIDEAN, COSINE). diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e4d79b2f..8fdac7b3 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -25,6 +25,8 @@ #include #include +#include "arrow/c/abi.h" +#include "paimon/cache/cache.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -45,7 +47,7 @@ class FileSystem; class PAIMON_EXPORT ReadContext { public: ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_schema, + const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, bool enable_predicate_filter, bool enable_prefetch, uint32_t prefetch_batch_count, @@ -56,7 +58,8 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::map& options, - PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config); + PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config, + const std::shared_ptr& cache); ~ReadContext(); const std::string& GetPath() const { @@ -75,8 +78,8 @@ class PAIMON_EXPORT ReadContext { return options_; } - const std::vector& GetReadSchema() const { - return read_schema_; + const std::vector& GetReadFieldNames() const { + return read_field_names_; } const std::vector& GetReadFieldIds() const { @@ -105,7 +108,7 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } - const std::optional& GetSpecificTableSchema() { + const std::optional& GetSpecificTableSchema() const { return table_schema_; } std::shared_ptr GetMemoryPool() const { @@ -126,10 +129,30 @@ class PAIMON_EXPORT ReadContext { return cache_config_; } + std::shared_ptr GetCache() const { + return cache_; + } + + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. + bool HasReadSchema() const { + return read_schema_ != nullptr && read_schema_->release != nullptr; + } + + /// Get the read schema as a mutable C ArrowSchema pointer. + /// ImportSchema will consume (release) the schema content. + ArrowSchema* GetReadSchema() { + return read_schema_.get(); + } + + /// Set the read schema from a C ArrowSchema unique_ptr and take ownership of + /// schema resources (released via ArrowSchema::release in destructor). + /// Called internally by ReadContextBuilder. + void SetReadSchema(std::unique_ptr schema); + private: std::string path_; std::string branch_; - std::vector read_schema_; + std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; bool enable_predicate_filter_; @@ -146,6 +169,9 @@ class PAIMON_EXPORT ReadContext { std::map options_; PrefetchCacheMode prefetch_cache_mode_; CacheConfig cache_config_; + std::shared_ptr cache_; + // Owns schema resources and releases ArrowSchema::release in destructor. + std::unique_ptr read_schema_; }; /// `ReadContextBuilder` used to build a `ReadContext`, has input validation. @@ -168,9 +194,9 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// @param read_field_names Vector of field names to read from the table. /// @return Reference to this builder for method chaining. - /// @note Currently supports top-level field selection. Future versions may support - /// nested field selection using ArrowSchema for more granular projection - ReadContextBuilder& SetReadSchema(const std::vector& read_field_names); + /// @note Currently supports top-level field selection. For nested field selection + /// use SetReadSchema(std::unique_ptr) instead. + ReadContextBuilder& SetReadFieldNames(const std::vector& read_field_names); /// Set the schema fields to read from the table. /// /// If not set, all fields from the table schema will be read. This is useful for @@ -179,12 +205,51 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// @param read_field_ids Vector of field ids to read from the table. /// @return Reference to this builder for method chaining. - /// @note Currently supports top-level field selection. Future versions may support - /// nested field selection using ArrowSchema for more granular projection. - /// @note SetReadFieldIds() and SetReadSchema() are mutually exclusive. - /// Calling both will ignore the read schema set by SetReadSchema(). + /// @note Currently supports top-level field selection. + /// @note SetReadFieldIds() and SetReadFieldNames() are mutually exclusive. + /// Calling both will ignore the read schema set by SetReadFieldNames(). ReadContextBuilder& SetReadFieldIds(const std::vector& read_field_ids); + /// Set the read Arrow Schema for nested column pruning. + /// + /// The read schema is an Arrow C Data Interface schema where STRUCT types + /// may contain only a subset of the original sub-fields, enabling nested column + /// pruning to reduce I/O. Field matching is based on field name: the system + /// looks up each field by name in the table schema and rebuilds the aligned + /// schema using the table schema's type and metadata. Metadata propagation + /// from the user-provided schema is whitelist-based: currently only + /// "paimon.map.selected-keys" is preserved and merged into the final aligned + /// schema. + /// + /// To prune map entries by key, attach metadata "paimon.map.selected-keys" + /// to the target map field in read schema. The value is a comma-separated + /// key list, for example: "k1,k2". Only map fields with string key type + /// (Arrow utf8) are supported. + /// + /// Example: + /// @code{.cpp} + /// auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + /// auto map_meta = arrow::KeyValueMetadata::Make( + /// {"paimon.map.selected-keys"}, {"k1,k2"}); + /// auto projected_schema = arrow::schema({ + /// arrow::field("id", arrow::int64()), + /// map_field->WithMetadata(map_meta), + /// }); + /// + /// auto c_schema = std::make_unique(); + /// arrow::ExportSchema(*projected_schema, c_schema.get()); + /// + /// ReadContextBuilder builder("/path/to/table"); + /// builder.SetReadSchema(std::move(c_schema)); + /// @endcode + /// + /// @param read_schema Arrow C Schema. Ownership of schema resources is transferred + /// to the built ReadContext. + /// @return Reference to this builder for method chaining. + /// @note Priority: read_schema > read_field_ids > read_field_names. + /// When set, read_field_ids and read_field_names are ignored. + ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); + /// Set a configuration options map to set some option entries which are not defined in the /// table schema or whose values you want to overwrite. /// @note The options map will clear the options added by `AddOption()` before. @@ -341,6 +406,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Inject a cache for read operations. Passing nullptr disables cache. + /// @return Reference to this builder for method chaining. + ReadContextBuilder& WithCache(const std::shared_ptr& cache); + /// Build and return a `ReadContext` instance with input validation. /// @return Result containing the constructed `ReadContext` or an error status. Result> Finish(); diff --git a/include/paimon/reader/batch_reader.h b/include/paimon/reader/batch_reader.h index 07835c43..c0033c20 100644 --- a/include/paimon/reader/batch_reader.h +++ b/include/paimon/reader/batch_reader.h @@ -43,6 +43,8 @@ class PAIMON_EXPORT BatchReader { /// If EOF is reached, returns an OK status with a nullptr array. Returns an error status only /// for critical failures (e.g., IO errors). Once an error is returned, this method must not be /// retried, as it will repeatedly return the same error code. + /// \note IMPORTANT: A non-EOF ArrowArray and all its nested child arrays must have offset 0 to + /// avoid potential issues during conversion through the Arrow C Data Interface. /// /// @return A result containing a `::ReadBatch`, which consists of a unique pointer to /// `ArrowArray` and a unique pointer to `ArrowSchema`. Returned array contains a `_VALUE_KIND` @@ -55,6 +57,8 @@ class PAIMON_EXPORT BatchReader { /// If EOF is reached, returns an OK status with a nullptr array. Returns an error status only /// for critical failures (e.g., IO errors). Once an error is returned, this method must not be /// retried, as it will repeatedly return the same error code. + /// \note IMPORTANT: A non-EOF ArrowArray and all its nested child arrays must have offset 0 to + /// avoid potential issues during conversion through the Arrow C Data Interface. /// /// @return A result containing a `::ReadBatch` and a valid bitmap. `::ReadBatch` consists of a /// unique pointer to `ArrowArray` and a unique pointer to `ArrowSchema`. Returned array diff --git a/include/paimon/reader/count_reader.h b/include/paimon/reader/count_reader.h new file mode 100644 index 00000000..88603e34 --- /dev/null +++ b/include/paimon/reader/count_reader.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// Reader abstraction for count queries. +class PAIMON_EXPORT CountReader { + public: + virtual ~CountReader() = default; + + /// Count rows for splits bound by the corresponding CreateCountReader call. + virtual Result CountRows() = 0; +}; + +} // namespace paimon diff --git a/include/paimon/reader/file_batch_reader.h b/include/paimon/reader/file_batch_reader.h index 38c89f00..87771be3 100644 --- a/include/paimon/reader/file_batch_reader.h +++ b/include/paimon/reader/file_batch_reader.h @@ -49,8 +49,14 @@ class PAIMON_EXPORT FileBatchReader : public BatchReader { using BatchReader::NextBatch; using BatchReader::NextBatchWithBitmap; - /// Get the row number of the first row in the previously read batch. - virtual Result GetPreviousBatchFirstRowNumber() const = 0; + /// Get the file-level row ID for a given batch-relative row index + /// in the previously read batch. + /// + /// @param batch_row_id Zero-based index within the current batch. + /// @return The corresponding file-level row ID, or Status::Invalid + /// if no batch has been read yet, the last batch was EOF, + /// or batch_row_id is out of range. + virtual Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const = 0; /// Get the number of rows in the file. virtual Result GetNumberOfRows() const = 0; diff --git a/include/paimon/result.h b/include/paimon/result.h index 68d2f4bc..bfba6ba5 100644 --- a/include/paimon/result.h +++ b/include/paimon/result.h @@ -273,11 +273,11 @@ inline Status GenericToStatus(Result&& res) { #define PAIMON_ASSIGN_OR_RAISE_IMPL(result_name, lhs, rexpr) \ auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).ok(), (result_name).status(), PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).value(); + lhs = std::move(result_name).value() #define PAIMON_ASSIGN_OR_RAISE_NAME(x, y) PAIMON_CONCAT(x, y) #define PAIMON_ASSIGN_OR_RAISE(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL(PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, \ - (rexpr)); + (rexpr)) } // namespace paimon diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 30644402..2dea6ac5 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -25,6 +25,7 @@ #include #include +#include "paimon/cache/cache.h" #include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -50,7 +51,9 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options); + const std::optional& table_schema, + const std::map& options, + const std::shared_ptr& cache); ~ScanContext(); @@ -88,6 +91,14 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + + std::shared_ptr GetCache() const { + return cache_; + } + private: std::string path_; bool is_streaming_mode_; @@ -97,7 +108,9 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::optional table_schema_; std::map options_; + std::shared_ptr cache_; }; /// Filter configuration for table scan operations @@ -180,6 +193,22 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// This optimization allows the scanner to use a pre-loaded schema instead of + /// reading it from the table metadata, which can improve performance especially + /// in scenarios with many small scan operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema will be loaded from the table path. + ScanContextBuilder& SetTableSchema(const std::string& table_schema); + + /// Inject a cache for scan operations. Passing nullptr disables cache. + /// @return Reference to this builder for method chaining. + ScanContextBuilder& WithCache(const std::shared_ptr& cache); + /// Build and return a `ScanContext` instance with input validation. /// @return Result containing the constructed `ScanContext` or an error status. Result> Finish(); diff --git a/include/paimon/table/source/data_split.h b/include/paimon/table/source/data_split.h index 865e785c..4de0d1cf 100644 --- a/include/paimon/table/source/data_split.h +++ b/include/paimon/table/source/data_split.h @@ -77,6 +77,9 @@ class PAIMON_EXPORT DataSplit : public Split { std::string ToString() const; }; + /// Get the bucket id of this data split. + virtual int32_t Bucket() const = 0; + /// Get the list of metadata for all data files in this split. /// @note This method will be removed in future versions and is only used for append tables. virtual std::vector GetFileList() const = 0; diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 157561f9..3bb6610a 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -26,6 +26,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" +#include "paimon/reader/count_reader.h" #include "paimon/result.h" #include "paimon/table/source/split.h" #include "paimon/visibility.h" @@ -66,6 +67,12 @@ class PAIMON_EXPORT TableRead { virtual Result> CreateReader( const std::shared_ptr& split) = 0; + /// Creates a `CountReader` for count queries on the specified splits. + /// + /// Implementations may override this to provide a more efficient count path. + virtual Result> CreateCountReader( + const std::vector>& splits); + protected: explicit TableRead(const std::shared_ptr& memory_pool); diff --git a/include/paimon/utils/roaring_bitmap32.h b/include/paimon/utils/roaring_bitmap32.h index 24b5c98e..de77ce53 100644 --- a/include/paimon/utils/roaring_bitmap32.h +++ b/include/paimon/utils/roaring_bitmap32.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -137,6 +138,10 @@ class PAIMON_EXPORT RoaringBitmap32 { Iterator End() const; /// @return the iterator moved to the value which is equal or larger than key Iterator EqualOrLarger(int32_t key) const; + /// @return the first value which is equal or larger than x + std::optional NextValue(int32_t x) const; + /// @return the largest value which is smaller than x + std::optional PreviousValue(int32_t x) const; /// Computes the intersection between two bitmaps and returns new bitmap. /// The current bitmap and the provided bitmap are unchanged. diff --git a/include/paimon/utils/row_range_index.h b/include/paimon/utils/row_range_index.h index f5351a0f..4a349492 100644 --- a/include/paimon/utils/row_range_index.h +++ b/include/paimon/utils/row_range_index.h @@ -34,7 +34,8 @@ class PAIMON_EXPORT RowRangeIndex { public: /// Creates a RowRangeIndex from the given ranges. The ranges will be sorted and merged /// (overlapping and adjacent ranges are combined) before indexing. - static Result Create(const std::vector& ranges); + static Result Create(const std::vector& ranges, + bool merge_adjacent = true); /// Returns the sorted, non-overlapping ranges held by this index. const std::vector& Ranges() const; @@ -42,6 +43,12 @@ class PAIMON_EXPORT RowRangeIndex { /// Returns true if any range in this index intersects with the interval [start, end]. bool Intersects(int64_t start, int64_t end) const; + /// Returns true if one range in this index fully contains `range`. + bool Contains(const Range& range) const; + + /// Returns true if one range in this index exactly equals `range`. + bool ContainsExactly(const Range& range) const; + /// Returns the sub-ranges of this index that intersect with the interval [start, end]. /// Each returned range is clipped to lie within [start, end]. std::vector IntersectedRanges(int64_t start, int64_t end) const; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 1c629386..f80c3ee2 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -27,6 +27,7 @@ set(PAIMON_COMMON_SRCS common/data/binary_string.cpp common/data/blob.cpp common/data/blob_descriptor.cpp + common/data/blob_view_struct.cpp common/data/blob_utils.cpp common/data/columnar/columnar_array.cpp common/data/columnar/columnar_map.cpp @@ -39,6 +40,24 @@ set(PAIMON_COMMON_SRCS common/data/serializer/row_compacted_serializer.cpp common/data/serializer/binary_serializer_utils.cpp common/data/timestamp.cpp + common/data/variant/generic_variant.cpp + common/data/variant/infer_variant_shredding_schema.cpp + common/data/variant/variant_shredding_inference_session.cpp + common/data/variant/variant.cpp + common/data/variant/variant_binary_util.cpp + common/data/variant/variant_builder.cpp + common/data/variant/variant_get.cpp + common/data/variant/variant_json_utils.cpp + common/data/variant/variant_path_segment.cpp + common/data/variant/variant_reassembler.cpp + common/data/variant/variant_shredding_batch_converter.cpp + common/data/variant/variant_access_utils.cpp + common/data/variant/variant_shredding_read_plan_factory.cpp + common/data/variant/variant_shredding_utils.cpp + common/data/variant/variant_shredding_write_plan.cpp + common/data/variant/variant_shredding_write_plan_factory.cpp + common/data/variant/variant_shredding_writer.cpp + common/data/variant/variant_type_utils.cpp common/defs.cpp common/executor/executor.cpp common/factories/singleton.cpp @@ -67,6 +86,7 @@ set(PAIMON_COMMON_SRCS common/io/data_output_stream.cpp common/io/memory_segment_output_stream.cpp common/io/offset_input_stream.cpp + common/io/cache/cache.cpp common/io/cache/cache_key.cpp common/io/cache/cache_manager.cpp common/io/cache/lru_cache.cpp @@ -104,6 +124,7 @@ set(PAIMON_COMMON_SRCS common/predicate/not_equal.cpp common/predicate/not_in.cpp common/predicate/or.cpp + common/predicate/predicate.cpp common/predicate/predicate_builder.cpp common/predicate/predicate_utils.cpp common/predicate/starts_with.cpp @@ -112,6 +133,8 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_fallback_batch_reader.cpp + common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp common/sst/block_handle.cpp @@ -137,6 +160,16 @@ set(PAIMON_COMMON_SRCS common/utils/bloom_filter64.cpp common/utils/crc32c.cpp common/utils/decimal_utils.cpp + common/data/shredding/map_shared_shredding_utils.cpp + common/data/shredding/map_shared_shredding_schema_utils.cpp + common/data/shredding/map_shared_shredding_write_plan_factory.cpp + common/data/shredding/shredding_write_plan_factories.cpp + common/data/shredding/map_shared_shredding_context.cpp + common/data/shredding/map_shared_shredding_batch_converter.cpp + common/data/shredding/map_shared_shredding_column_allocator.cpp + common/data/shredding/lru_map_shared_shredding_column_allocator.cpp + common/data/shredding/map_shared_shredding_file_reader.cpp + common/data/shredding/shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -149,6 +182,12 @@ set(PAIMON_COMMON_SRCS common/utils/status.cpp common/utils/string_utils.cpp) +if(PAIMON_ENABLE_S3) + list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp + common/fs/object_store_file_system.cpp) + set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl) +endif() + set(PAIMON_CORE_SRCS core/disk/file_io_channel.cpp core/mergetree/spill_reader.cpp @@ -161,6 +200,7 @@ set(PAIMON_CORE_SRCS core/bucket/hive_bucket_function.cpp core/bucket/mod_bucket_function.cpp core/bucket/bucket_id_calculator.cpp + core/casting/binary_to_blob_cast_executor.cpp core/casting/binary_to_string_cast_executor.cpp core/casting/boolean_to_decimal_cast_executor.cpp core/casting/boolean_to_numeric_cast_executor.cpp @@ -203,6 +243,7 @@ set(PAIMON_CORE_SRCS core/index/index_file_handler.cpp core/index/global_index_meta.cpp core/index/index_file_meta_serializer.cpp + core/io/generic_row_to_arrow_array_converter.cpp core/io/meta_to_arrow_array_converter.cpp core/io/async_key_value_producer_and_consumer.cpp core/io/data_file_meta_09_serializer.cpp @@ -212,17 +253,23 @@ set(PAIMON_CORE_SRCS core/io/data_file_meta.cpp core/io/data_file_meta_serializer.cpp core/io/data_file_path_factory.cpp + core/io/append_data_file_writer_factory.cpp + core/io/blob_data_file_writer_factory.cpp + core/io/data_file_writer_factory.cpp core/io/data_file_writer.cpp core/io/field_mapping_reader.cpp core/io/complete_row_tracking_fields_reader.cpp core/io/file_index_evaluator.cpp core/io/key_value_data_file_record_reader.cpp + core/io/key_value_data_file_writer_factory.cpp core/io/key_value_data_file_writer.cpp core/io/key_value_in_memory_record_reader.cpp core/io/merged_key_value_record_reader.cpp core/io/key_value_meta_projection_consumer.cpp core/io/key_value_projection_consumer.cpp core/io/key_value_projection_reader.cpp + core/io/shredding_append_data_file_writer_factory.cpp + core/io/shredding_key_value_data_file_writer_factory.cpp core/io/multiple_blob_file_writer.cpp core/io/rolling_blob_file_writer.cpp core/manifest/file_kind.cpp @@ -263,6 +310,7 @@ set(PAIMON_CORE_SRCS core/mergetree/lookup_file.cpp core/mergetree/lookup_levels.cpp core/mergetree/lookup/remote_lookup_file_manager.cpp + core/mergetree/row_count_accumulator.cpp core/migrate/file_meta_utils.cpp core/operation/data_evolution_file_store_scan.cpp core/operation/data_evolution_split_read.cpp @@ -270,6 +318,17 @@ set(PAIMON_CORE_SRCS core/operation/abstract_split_read.cpp core/operation/append_only_file_store_scan.cpp core/operation/append_only_file_store_write.cpp + core/operation/metrics/commit_metrics.cpp + core/operation/commit/conflict_detection.cpp + core/operation/commit/commit_scanner.cpp + core/operation/commit/commit_changes_provider.cpp + core/operation/commit/compacted_changelog_path_resolver.cpp + core/operation/commit/overwrite_changes_provider.cpp + core/operation/commit/row_id_column_conflict_checker.cpp + core/operation/commit/manifest_entry_changes.cpp + core/operation/commit/row_tracking_commit_utils.cpp + core/operation/commit/sequence_snapshot_properties.cpp + core/operation/commit/retry_waiter.cpp core/operation/commit_context.cpp core/operation/expire_snapshots.cpp core/operation/file_store_commit.cpp @@ -287,6 +346,7 @@ set(PAIMON_CORE_SRCS core/operation/raw_file_split_read.cpp core/operation/read_context.cpp core/operation/scan_context.cpp + core/manifest/snapshot_live_manifest_entries.cpp core/operation/write_context.cpp core/operation/write_restore.cpp core/postpone/postpone_bucket_writer.cpp @@ -301,9 +361,11 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp core/table/table.cpp + core/table/bucket_mode.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp + core/table/source/append_count_reader.cpp core/table/source/append_only_table_read.cpp core/table/source/split.cpp core/table/source/data_split_impl.cpp @@ -311,6 +373,7 @@ set(PAIMON_CORE_SRCS core/table/source/data_table_stream_scan.cpp core/table/source/fallback_table_read.cpp core/table/source/key_value_table_read.cpp + core/table/source/pk_count_reader.cpp core/table/source/merge_tree_split_generator.cpp core/table/source/data_evolution_split_generator.cpp core/table/source/plan_impl.cpp @@ -321,12 +384,19 @@ set(PAIMON_CORE_SRCS core/table/source/data_evolution_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp - core/table/system/options_system_table.cpp + core/table/system/global_system_tables.cpp + core/table/system/in_memory_system_table.cpp + core/table/system/metadata_system_tables.cpp + core/table/system/read_optimized_system_table.cpp core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp core/tag/tag.cpp + core/utils/branch_manager.cpp + core/utils/blob_view_lookup.cpp + core/utils/consumer_manager.cpp core/utils/field_mapping.cpp + core/utils/nested_projection_utils.cpp core/utils/file_store_path_factory.cpp core/utils/file_utils.cpp core/utils/manifest_meta_reader.cpp @@ -349,16 +419,17 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON + ${PAIMON_OBJECT_STORE_LINK_LIBS} STATIC_LINK_LIBS arrow tbb glog - dl fmt roaring_bitmap xxhash Threads::Threads RapidJSON + ${PAIMON_OBJECT_STORE_LINK_LIBS} SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) @@ -401,7 +472,18 @@ if(PAIMON_BUILD_TESTS) common/data/timestamp_test.cpp common/data/blob_test.cpp common/data/blob_descriptor_test.cpp + common/data/blob_view_struct_test.cpp common/data/blob_utils_test.cpp + common/data/variant/generic_variant_test.cpp + common/data/variant/infer_variant_shredding_schema_test.cpp + common/data/variant/variant_access_utils_test.cpp + common/data/variant/variant_shredding_read_plan_factory_test.cpp + common/data/variant/variant_shredding_write_plan_factory_test.cpp + common/data/variant/variant_get_test.cpp + common/data/variant/variant_json_utils_test.cpp + common/data/variant/variant_shredding_test.cpp + common/data/variant/variant_test.cpp + common/data/variant/variant_type_utils_test.cpp common/executor/default_executor_test.cpp common/format/column_stats_test.cpp common/fs/external_path_provider_test.cpp @@ -463,6 +545,8 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_fallback_batch_reader_test.cpp + common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp common/reader/data_evolution_row_test.cpp @@ -519,6 +603,15 @@ if(PAIMON_BUILD_TESTS) common/utils/decimal_utils_test.cpp common/utils/threadsafe_queue_test.cpp common/utils/generic_lru_cache_test.cpp + common/data/shredding/map_shared_shredding_utils_test.cpp + common/data/shredding/map_shared_shredding_schema_utils_test.cpp + common/data/shredding/map_shared_shredding_batch_converter_test.cpp + common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp + common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp + common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp + common/data/shredding/map_shared_shredding_field_dict_test.cpp + common/data/shredding/map_shared_shredding_context_test.cpp + common/data/shredding/map_shared_shredding_file_reader_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static @@ -545,9 +638,7 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) add_paimon_test(core_test @@ -587,6 +678,7 @@ if(PAIMON_BUILD_TESTS) core/index/index_file_meta_serializer_test.cpp core/index/index_file_handler_test.cpp core/io/compact_increment_test.cpp + core/io/infer_shredding_file_writer_test.cpp core/io/concat_key_value_record_reader_test.cpp core/io/data_file_meta_serializer_test.cpp core/io/data_file_path_factory_test.cpp @@ -661,13 +753,25 @@ if(PAIMON_BUILD_TESTS) core/mergetree/spill_reader_writer_test.cpp core/migrate/file_meta_utils_test.cpp core/operation/metrics/compaction_metrics_test.cpp + core/operation/metrics/commit_stats_test.cpp core/operation/data_evolution_file_store_scan_test.cpp core/operation/data_evolution_split_read_test.cpp + core/operation/commit/compacted_changelog_path_resolver_test.cpp + core/operation/commit/commit_changes_provider_test.cpp + core/operation/commit/commit_scanner_test.cpp + core/operation/commit/conflict_detection_test.cpp + core/operation/commit/manifest_entry_changes_test.cpp + core/operation/commit/overwrite_changes_provider_test.cpp + core/operation/commit/row_id_column_conflict_checker_test.cpp + core/operation/commit/row_tracking_commit_utils_test.cpp + core/operation/commit/sequence_snapshot_properties_test.cpp + core/operation/commit/retry_waiter_test.cpp core/operation/key_value_file_store_write_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp core/operation/commit_metrics_test.cpp + core/operation/commit_context_test.cpp core/operation/expire_snapshots_test.cpp core/operation/file_store_commit_impl_test.cpp core/operation/file_store_commit_test.cpp @@ -696,20 +800,28 @@ if(PAIMON_BUILD_TESTS) core/stats/simple_stats_collector_test.cpp core/stats/simple_stats_test.cpp core/table/table_test.cpp + core/table/bucket_mode_test.cpp core/table/sink/commit_message_test.cpp core/table/sink/commit_message_impl_test.cpp core/table/source/fallback_data_split_test.cpp core/table/source/table_read_test.cpp + core/table/source/append_count_reader_test.cpp + core/table/source/pk_count_reader_test.cpp + core/table/source/data_evolution_batch_scan_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp + core/table/source/snapshot/snapshot_reader_test.cpp core/table/source/startup_mode_test.cpp core/table/source/table_scan_test.cpp core/table/system/system_table_test.cpp core/tag/tag_test.cpp + core/utils/blob_view_lookup_test.cpp core/utils/branch_manager_test.cpp + core/utils/consumer_manager_test.cpp core/utils/file_store_path_factory_cache_test.cpp core/utils/field_mapping_test.cpp + core/utils/nested_projection_utils_test.cpp core/utils/file_store_path_factory_test.cpp core/utils/file_utils_test.cpp core/utils/manifest_meta_reader_test.cpp @@ -725,20 +837,42 @@ if(PAIMON_BUILD_TESTS) ${TEST_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) + set(PAIMON_JINDO_FS_TEST_SOURCES) + set(PAIMON_JINDO_FS_TEST_LINK_LIBS) + if(PAIMON_ENABLE_JINDO) + # These unit tests do not access remote OSS. + list(APPEND PAIMON_JINDO_FS_TEST_SOURCES fs/jindo/jindo_file_system_unit_test.cpp + fs/jindo/jindo_utils_test.cpp) + if(PAIMON_ENABLE_NETWORK_TESTS) + # Factory and integration tests access real OSS. + list(APPEND PAIMON_JINDO_FS_TEST_SOURCES + fs/jindo/jindo_file_system_factory_test.cpp + fs/jindo/jindo_file_system_test.cpp) + endif() + list(APPEND PAIMON_JINDO_FS_TEST_LINK_LIBS + ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS}) + endif() + + set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES) + if(PAIMON_ENABLE_S3) + list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES + common/fs/object_store_file_system_test.cpp) + endif() + add_paimon_test(fs_test SOURCES common/fs/file_system_test.cpp common/fs/resolving_file_system_test.cpp + ${PAIMON_OBJECT_STORE_FS_TEST_SOURCES} fs/local/local_file_test.cpp - # fs/jindo/jindo_file_system_factory_test.cpp - # fs/jindo/jindo_file_system_test.cpp + ${PAIMON_JINDO_FS_TEST_SOURCES} STATIC_LINK_LIBS paimon_shared - "-Wl,--whole-archive" - paimon_local_file_system_static - # paimon_jindo_file_system_static - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} + ${PAIMON_JINDO_FS_TEST_LINK_LIBS} test_utils_static - ${GTEST_LINK_TOOLCHAIN}) + ${GTEST_LINK_TOOLCHAIN} + EXTRA_INCLUDES + ${JINDOSDK_INCLUDE_DIR}) endif() diff --git a/src/paimon/common/catalog/catalog_context.h b/src/paimon/common/catalog/catalog_context.h new file mode 100644 index 00000000..fecc3e01 --- /dev/null +++ b/src/paimon/common/catalog/catalog_context.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once +#include +#include + +#include "paimon/fs/file_system.h" + +namespace paimon { +struct CatalogContext { + CatalogContext(const std::string& _root_path, + const std::map& _options, + const std::shared_ptr& _file_system) + : root_path(_root_path), options(_options), file_system(_file_system) {} + + std::string root_path; + std::map options; + std::shared_ptr file_system; +}; + +} // namespace paimon diff --git a/src/paimon/common/compression/block_compression_factory_test.cpp b/src/paimon/common/compression/block_compression_factory_test.cpp index 107d20a3..90ad4cce 100644 --- a/src/paimon/common/compression/block_compression_factory_test.cpp +++ b/src/paimon/common/compression/block_compression_factory_test.cpp @@ -117,4 +117,35 @@ TEST_P(CompressionFactoryTest, TestCompressInsufficientOutputBuffer) { INSTANTIATE_TEST_SUITE_P(BlockCompressionTypeGroup, CompressionFactoryTest, ::testing::Values(BlockCompressionType::LZ4, BlockCompressionType::ZSTD)); +TEST(CompressionFactoryCreateTest, TestCreateFromCompressOptions) { + ASSERT_OK_AND_ASSIGN(auto none_factory, + BlockCompressionFactory::Create(CompressOptions{"none", 0})); + ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType()); + + // Codec name matching is case-insensitive. + ASSERT_OK_AND_ASSIGN(auto zstd_factory, + BlockCompressionFactory::Create(CompressOptions{"ZSTD", 1})); + ASSERT_EQ(BlockCompressionType::ZSTD, zstd_factory->GetCompressionType()); + + ASSERT_OK_AND_ASSIGN(auto lz4_factory, + BlockCompressionFactory::Create(CompressOptions{"Lz4", 0})); + ASSERT_EQ(BlockCompressionType::LZ4, lz4_factory->GetCompressionType()); + + // Unsupported codec name returns an Invalid status. + auto unsupported = BlockCompressionFactory::Create(CompressOptions{"lzo", 0}); + ASSERT_NOK(unsupported); + ASSERT_TRUE(unsupported.status().IsInvalid()); +} + +TEST(CompressionFactoryCreateTest, TestCreateFromCompressionType) { + ASSERT_OK_AND_ASSIGN(auto none_factory, + BlockCompressionFactory::Create(BlockCompressionType::NONE)); + ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType()); + + // LZO is declared but not yet implemented, so it hits the default branch. + auto lzo = BlockCompressionFactory::Create(BlockCompressionType::LZO); + ASSERT_NOK(lzo); + ASSERT_TRUE(lzo.status().IsInvalid()); +} + } // namespace paimon::test diff --git a/src/paimon/common/data/abstract_binary_writer.h b/src/paimon/common/data/abstract_binary_writer.h index 7565543a..fcbb396b 100644 --- a/src/paimon/common/data/abstract_binary_writer.h +++ b/src/paimon/common/data/abstract_binary_writer.h @@ -24,7 +24,7 @@ #include #include "paimon/common/data/binary_writer.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class BinaryArray; diff --git a/src/paimon/common/data/binary_array.cpp b/src/paimon/common/data/binary_array.cpp index de195b8c..f92e89b7 100644 --- a/src/paimon/common/data/binary_array.cpp +++ b/src/paimon/common/data/binary_array.cpp @@ -25,8 +25,8 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_data_read_utils.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" namespace paimon { diff --git a/src/paimon/common/data/binary_array_writer.h b/src/paimon/common/data/binary_array_writer.h index 6b214d92..626f996b 100644 --- a/src/paimon/common/data/binary_array_writer.h +++ b/src/paimon/common/data/binary_array_writer.h @@ -23,7 +23,7 @@ #include "arrow/api.h" #include "paimon/common/data/abstract_binary_writer.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class BinaryArray; class MemoryPool; diff --git a/src/paimon/common/data/binary_map.h b/src/paimon/common/data/binary_map.h index 71694617..4783833f 100644 --- a/src/paimon/common/data/binary_map.h +++ b/src/paimon/common/data/binary_map.h @@ -20,8 +20,8 @@ #pragma once #include "paimon/common/data/binary_array.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/memory/memory_segment.h" namespace paimon { /// A binary implementation of `InternalMap` which is backed by a single `MemorySegment`. /// Binary layout: [4 byte(keyArray size in bytes)] + [Key BinaryArray] + [Value BinaryArray]. diff --git a/src/paimon/common/data/binary_row.cpp b/src/paimon/common/data/binary_row.cpp index 1cdc66bf..284b095f 100644 --- a/src/paimon/common/data/binary_row.cpp +++ b/src/paimon/common/data/binary_row.cpp @@ -23,11 +23,11 @@ #include "fmt/format.h" #include "paimon/common/data/binary_data_read_utils.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" namespace paimon { const int64_t BinaryRow::FIRST_BYTE_ZERO = diff --git a/src/paimon/common/data/binary_row.h b/src/paimon/common/data/binary_row.h index a8640632..e3dfe8b0 100644 --- a/src/paimon/common/data/binary_row.h +++ b/src/paimon/common/data/binary_row.h @@ -159,6 +159,19 @@ struct hash> { } }; +/// for std::unordered_map, ...> +template <> +struct hash> { + size_t operator()( + const std::tuple& partition_bucket_level) const { + const auto& [partition, bucket, level] = partition_bucket_level; + size_t hash = paimon::MurmurHashUtils::HashUnsafeBytes( + reinterpret_cast(&bucket), 0, sizeof(bucket), partition.HashCode()); + return paimon::MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&level), 0, + sizeof(level), hash); + } +}; + template <> struct hash { size_t operator()(const paimon::BinaryRow& row) const { diff --git a/src/paimon/common/data/binary_row_test.cpp b/src/paimon/common/data/binary_row_test.cpp index 17ee3331..d851881a 100644 --- a/src/paimon/common/data/binary_row_test.cpp +++ b/src/paimon/common/data/binary_row_test.cpp @@ -30,7 +30,6 @@ #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/serializer/binary_row_serializer.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" @@ -39,6 +38,7 @@ #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -341,8 +341,9 @@ TEST_F(BinaryRowTest, TestBinary) { auto pool = GetDefaultPool(); BinaryRow row(2); BinaryRowWriter writer(&row, 0, pool.get()); - char chars1[3] = {1, -1, 5}; - char chars2[8] = {1, -1, 5, 5, 1, 5, 1, 5}; + // explicit cast to avoid -Wnarrowing on platforms where char is unsigned (e.g. aarch64) + char chars1[3] = {1, static_cast(-1), 5}; + char chars2[8] = {1, static_cast(-1), 5, 5, 1, 5, 1, 5}; std::string str1(chars1, 3); std::string str2(chars2, 8); Bytes bytes1(str1, pool.get()); diff --git a/src/paimon/common/data/binary_row_writer.cpp b/src/paimon/common/data/binary_row_writer.cpp index c9b5ce05..462a8561 100644 --- a/src/paimon/common/data/binary_row_writer.cpp +++ b/src/paimon/common/data/binary_row_writer.cpp @@ -29,12 +29,12 @@ #include "arrow/util/checked_cast.h" #include "fmt/format.h" #include "paimon/common/data/binary_string.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" namespace paimon { @@ -143,7 +143,7 @@ Result BinaryRowWriter::CreateFieldSetter( }; return field_setter; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/data/binary_row_writer.h b/src/paimon/common/data/binary_row_writer.h index 954ba1d6..875d0341 100644 --- a/src/paimon/common/data/binary_row_writer.h +++ b/src/paimon/common/data/binary_row_writer.h @@ -26,9 +26,9 @@ #include "paimon/common/data/abstract_binary_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/types/row_kind.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace arrow { diff --git a/src/paimon/common/data/binary_section.h b/src/paimon/common/data/binary_section.h index 73eeeb67..74ab46b0 100644 --- a/src/paimon/common/data/binary_section.h +++ b/src/paimon/common/data/binary_section.h @@ -23,9 +23,9 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/data/binary_string.h b/src/paimon/common/data/binary_string.h index 2de9b588..a9d71aff 100644 --- a/src/paimon/common/data/binary_string.h +++ b/src/paimon/common/data/binary_string.h @@ -29,7 +29,7 @@ #include #include "paimon/common/data/binary_section.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/data/blob.cpp b/src/paimon/common/data/blob.cpp index 66953273..0c706aaa 100644 --- a/src/paimon/common/data/blob.cpp +++ b/src/paimon/common/data/blob.cpp @@ -97,8 +97,20 @@ Result> Blob::NewInputStream( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file, fs->Open(impl_->GetDescriptor()->Uri())); - return OffsetInputStream::Create(std::move(file), impl_->GetDescriptor()->Length(), - impl_->GetDescriptor()->Offset()); + int64_t blob_length = impl_->GetDescriptor()->Length(); + int64_t blob_offset = impl_->GetDescriptor()->Offset(); + + PAIMON_ASSIGN_OR_RAISE(int64_t total_length, file->Length()); + if (PAIMON_UNLIKELY(blob_offset > total_length)) { + return Status::Invalid( + fmt::format("offset {} exceed total length {}", blob_offset, total_length)); + } + if (blob_length == -1) { + // blob_length == -1 means it's dynamic length, should read to the end + blob_length = total_length - blob_offset; + } + + return OffsetInputStream::Create(std::move(file), blob_length, blob_offset, total_length); } Result> Blob::ToData(const std::shared_ptr& fs, @@ -108,8 +120,9 @@ Result> Blob::ToData(const std::shared_ptr& } Result> Blob::ArrowField( - const std::string& field_name, std::unordered_map metadata) { - auto blob_field = BlobUtils::ToArrowField(field_name, /*nullable=*/false, metadata); + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + auto blob_field = BlobUtils::ToArrowField(field_name, nullable, metadata); auto field = std::make_unique<::ArrowSchema>(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*blob_field, field.get())); return field; diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index 6a478e70..deacd1d0 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -20,6 +20,10 @@ #pragma once #include +#include +#include +#include +#include namespace paimon { @@ -48,6 +52,64 @@ class BlobDefs { /// A bin_length value of -1 in the index indicates a null blob entry. static constexpr int64_t kNullBinLength = -1; + /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by + /// data-evolution partial updates for rows whose blob value is not updated. A placeholder + /// entry occupies no file space; readers must fall back to an older blob file covering the + /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH. + static constexpr int64_t kPlaceholderBinLength = -2; + /// Sentinel bytes standing for a placeholder blob value in two internal channels: + /// + /// - Write channel: a data-evolution partial update (a blob-only column write, see + /// kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format + /// writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build + /// such write arrays. Outside that mode the writer never interprets values, so arbitrary + /// user bytes can never be turned into a placeholder entry. + /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these + /// bytes for -2 entries so the fallback merge can identify placeholders after the batch + /// has passed through schema-mapping readers. + /// + /// Both channels identify a placeholder by exact byte equality with this internal reserved + /// value (IsPlaceholderSentinel), and the fallback merge byte-compares every layer of a + /// bunch — including files written outside the write channel. A user blob whose bytes + /// exactly equal the marker therefore collides with it in two ways: written through the + /// partial-update channel it is persisted as a placeholder entry, which a single-layer read + /// rejects loudly (no older layer can resolve it); left untouched in an older layer under a + /// later partial update it reads as a placeholder in every layer and silently degrades to a + /// null blob. The marker is distinctive enough that these collisions are accepted as + /// negligibly improbable. Sentinel bytes are never stored in blob files. + static constexpr char kPlaceholderSentinel[] = "_PAIMON_BLOB_PLACEHOLDER"; + /// Byte length of kPlaceholderSentinel, excluding the literal's terminating NUL. + static constexpr int32_t kPlaceholderSentinelLength = sizeof(kPlaceholderSentinel) - 1; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// reader emits kPlaceholderSentinel for placeholder entries instead of failing on them. + /// Only the data-evolution blob fallback read path sets this. + static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel"; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// format writer persists a value exactly equal to kPlaceholderSentinel as a bin_length -2 + /// entry. Only set for data-evolution partial updates, i.e. blob-only column writes of a + /// table with data evolution enabled; all other writes store bytes verbatim. + static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder"; + + /// The sentinel bytes for building a data-evolution partial-update write array: a row equal + /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey). + static std::string_view PlaceholderSentinelView() { + return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)}; + } + + /// True when the bytes are exactly the placeholder sentinel. + static bool IsPlaceholderSentinel(const char* data, size_t size) { + return size == static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } + + /// Removes the internal placeholder option keys from a format options map. The placeholder + /// channels must only ever be enabled by the internal data-evolution write and read paths, + /// so every consumer building format options from user-supplied table options strips these + /// keys before applying its own decision. + static void EraseInternalPlaceholderOptions(std::map* options) { + options->erase(kEmitPlaceholderSentinelKey); + options->erase(kWritePlaceholderKey); + } /// Blob file format version. static constexpr int8_t kFileVersion = 1; /// Magic number identifying the start of each blob bin. @@ -56,8 +118,8 @@ class BlobDefs { static constexpr int32_t kContentStartOffset = 4; /// Total metadata length per bin: magic(4) + bin_length(8) + crc32(4) = 16. static constexpr int32_t kTotalMetaLength = 16; - /// Blob file header length: index_len(4) + version(1) = 5. - static constexpr uint32_t kBlobFileHeaderLength = 5; + /// Blob file footer length: index_len(4) + version(1) = 5. + static constexpr uint32_t kBlobFileFooterLength = 5; }; } // namespace paimon diff --git a/src/paimon/common/data/blob_descriptor.cpp b/src/paimon/common/data/blob_descriptor.cpp index f5624c78..c262ebf1 100644 --- a/src/paimon/common/data/blob_descriptor.cpp +++ b/src/paimon/common/data/blob_descriptor.cpp @@ -53,7 +53,7 @@ Result> BlobDescriptor::Create(int8_t version, PAIMON_UNIQUE_PTR BlobDescriptor::Serialize(const std::shared_ptr& pool) const { MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); out.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); - out.WriteValue(version_); + out.WriteValue(kCurrentVersion); out.WriteValue(kMagic); out.WriteValue(static_cast(uri_.size())); diff --git a/src/paimon/common/data/blob_descriptor.h b/src/paimon/common/data/blob_descriptor.h index 6664b85e..ea41c97d 100644 --- a/src/paimon/common/data/blob_descriptor.h +++ b/src/paimon/common/data/blob_descriptor.h @@ -41,7 +41,7 @@ namespace paimon { /// | 13 + N | offset | long | 8 | /// | 21 + N | length | long | 8 | -class BlobDescriptor { +class PAIMON_EXPORT BlobDescriptor { public: static Result> Create(const std::string& uri, int64_t offset, int64_t length); diff --git a/src/paimon/common/data/blob_descriptor_test.cpp b/src/paimon/common/data/blob_descriptor_test.cpp index a5221bf9..1fbd9de0 100644 --- a/src/paimon/common/data/blob_descriptor_test.cpp +++ b/src/paimon/common/data/blob_descriptor_test.cpp @@ -59,6 +59,10 @@ TEST_F(BlobDescriptorTest, TestDeserializeCompatibilityForJavaWithVersion1) { ASSERT_EQ(descriptor->Uri(), "test_uri"); ASSERT_EQ(descriptor->Offset(), 1024); ASSERT_EQ(descriptor->Length(), 2048); + + PAIMON_UNIQUE_PTR cpp_serialized = descriptor->Serialize(pool_); + auto cpp_serialized_string = std::string(cpp_serialized->data(), cpp_serialized->size()); + ASSERT_NE(cpp_serialized_string, java_serialized); } TEST_F(BlobDescriptorTest, TestDeserializeCompatibilityForJavaWithVersion2) { diff --git a/src/paimon/common/data/blob_test.cpp b/src/paimon/common/data/blob_test.cpp index 4c95711c..aa20c6e5 100644 --- a/src/paimon/common/data/blob_test.cpp +++ b/src/paimon/common/data/blob_test.cpp @@ -119,7 +119,7 @@ TEST_F(BlobTest, TestNewInputStreamWithOffsetAndLength) { ASSERT_OK_AND_ASSIGN(auto input_stream, blob->NewInputStream(file_system_)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); ASSERT_EQ(6, length); // Test reading with offset and length applied @@ -136,7 +136,7 @@ TEST_F(BlobTest, TestNewInputStreamWithDynamicLength) { ASSERT_OK_AND_ASSIGN(auto input_stream, blob->NewInputStream(file_system_)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); ASSERT_EQ(12, length); // Test reading from offset to end (should read "cdefghijklmn") @@ -147,38 +147,34 @@ TEST_F(BlobTest, TestNewInputStreamWithDynamicLength) { } TEST_F(BlobTest, TestArrowField) { - { - // basic: field name, non-nullable by default - ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("my_blob")); + for (bool nullable : {false, true}) { + ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("my_blob", nullable)); ASSERT_NE(schema, nullptr); - // import back to arrow::Field to verify auto field_result = arrow::ImportField(schema.get()); ASSERT_TRUE(field_result.ok()); auto field = field_result.ValueUnsafe(); ASSERT_EQ(field->name(), "my_blob"); ASSERT_EQ(field->type()->id(), arrow::Type::LARGE_BINARY); - ASSERT_FALSE(field->nullable()); + ASSERT_EQ(field->nullable(), nullable); ASSERT_TRUE(field->HasMetadata()); auto extension_type = field->metadata()->Get("paimon.extension.type"); ASSERT_TRUE(extension_type.ok()); ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob"); } { - // with custom metadata std::unordered_map custom_metadata = { {"custom_key", "custom_value"}}; - ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("meta_blob", custom_metadata)); + ASSERT_OK_AND_ASSIGN(auto schema, + Blob::ArrowField("meta_blob", /*nullable=*/false, custom_metadata)); auto field = arrow::ImportField(schema.get()).ValueUnsafe(); ASSERT_EQ(field->name(), "meta_blob"); ASSERT_FALSE(field->nullable()); ASSERT_TRUE(field->HasMetadata()); - // blob extension metadata should be present auto extension_type = field->metadata()->Get("paimon.extension.type"); ASSERT_TRUE(extension_type.ok()); ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob"); - // custom metadata should also be present auto custom_val = field->metadata()->Get("custom_key"); ASSERT_TRUE(custom_val.ok()); ASSERT_EQ(custom_val.ValueUnsafe(), "custom_value"); diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index 9eb716a1..41df10b9 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -20,65 +20,77 @@ #include "paimon/common/data/blob_utils.h" #include -#include +#include #include #include "arrow/api.h" #include "arrow/array/array_nested.h" #include "arrow/type.h" +#include "fmt/format.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" - namespace arrow { class Array; } namespace paimon { - BlobUtils::SeparatedSchemas BlobUtils::SeparateBlobSchema( - const std::shared_ptr& schema) { - std::vector> remaining_fields; + const std::shared_ptr& schema, const std::set& inline_fields) { + std::vector> main_fields; std::vector> blob_fields; - for (auto i = 0; i < schema->num_fields(); i++) { + for (int32_t i = 0; i < schema->num_fields(); i++) { auto field = schema->field(i); - if (IsBlobField(field)) { + if (IsBlobField(field) && inline_fields.count(field->name()) == 0) { + // Non-inline BLOB -> goes to blob file blob_fields.emplace_back(field); } else { - remaining_fields.emplace_back(field); + // Non-blob fields OR inline BLOB fields -> stay in main + main_fields.emplace_back(field); } } SeparatedSchemas result; - result.main_schema = arrow::schema(remaining_fields); + result.main_schema = arrow::schema(main_fields); result.blob_schema = arrow::schema(blob_fields); return result; } Result BlobUtils::SeparateBlobArray( - const std::shared_ptr& struct_array) { + const std::shared_ptr& struct_array, + const std::set& inline_fields) { std::shared_ptr old_type = std::static_pointer_cast(struct_array->type()); const auto& old_fields = old_type->fields(); const auto& old_arrays = struct_array->fields(); - std::vector> remaining_fields; - std::vector> remaining_arrays; - std::vector> blob_fields; - std::vector> blob_arrays; + arrow::ArrayVector main_arrays; + arrow::ArrayVector blob_arrays; + arrow::FieldVector main_fields; + arrow::FieldVector blob_fields; for (size_t i = 0; i < old_fields.size(); i++) { - if (IsBlobField(old_fields[i])) { + if (IsBlobField(old_fields[i]) && inline_fields.count(old_fields[i]->name()) == 0) { blob_fields.push_back(old_fields[i]); blob_arrays.push_back(old_arrays[i]); } else { - remaining_fields.push_back(old_fields[i]); - remaining_arrays.push_back(old_arrays[i]); + main_fields.push_back(old_fields[i]); + main_arrays.push_back(old_arrays[i]); } } + if (blob_fields.empty()) { + return Status::Invalid( + "SeparateBlobArray expects at least one non-inline blob field, but got none."); + } + SeparatedStructArrays result; - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.main_array, - arrow::StructArray::Make(remaining_arrays, remaining_fields)); + if (!main_fields.empty()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.main_array, + arrow::StructArray::Make(main_arrays, main_fields)); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.blob_array, arrow::StructArray::Make(blob_arrays, blob_fields)); return result; @@ -117,4 +129,69 @@ std::shared_ptr BlobUtils::ToArrowField( return arrow::field(field_name, arrow::large_binary(), nullable, std::make_shared(metadata)); } + +Status BlobUtils::ValidateBlobInlineFields(const std::shared_ptr& struct_array, + const std::set& field_names, + const std::string& config_label) { + if (field_names.empty()) { + return Status::OK(); + } + if (!struct_array) { + return Status::Invalid("array in ValidateBlobInlineFields must be a struct_array"); + } + + bool is_descriptor = (config_label == "blob-descriptor-field"); + for (const auto& field_name : field_names) { + auto field_array = struct_array->GetFieldByName(field_name); + if (!field_array) { + continue; + } + const auto* binary_array = + arrow::internal::checked_cast(field_array.get()); + if (!binary_array) { + return Status::Invalid( + fmt::format("cannot cast array for field {} to LargeBinaryArray", field_name)); + } + for (int64_t row = 0; row < binary_array->length(); ++row) { + if (binary_array->IsNull(row)) { + continue; + } + auto value = binary_array->GetView(row); + Result valid = is_descriptor + ? BlobDescriptor::IsBlobDescriptor(value.data(), value.size()) + : BlobViewStruct::IsBlobViewStruct(value.data(), value.size()); + PAIMON_ASSIGN_OR_RAISE(bool is_valid, std::move(valid)); + if (!is_valid) { + return Status::Invalid(fmt::format( + "BLOB inline field {} require values to be set as corresponding type.", + field_name)); + } + } + } + return Status::OK(); +} + +std::vector BlobUtils::ConvertBlobInlineDataFields( + const std::vector& data_fields, const std::vector& blob_inline_fields) { + if (blob_inline_fields.empty()) { + return data_fields; + } + + std::set blob_inline_field_set(blob_inline_fields.begin(), + blob_inline_fields.end()); + std::vector converted_fields; + converted_fields.reserve(data_fields.size()); + for (const auto& data_field : data_fields) { + if (blob_inline_field_set.find(data_field.Name()) == blob_inline_field_set.end()) { + converted_fields.push_back(data_field); + continue; + } + + auto binary_field = arrow::field(data_field.Name(), arrow::binary(), data_field.Nullable(), + data_field.ArrowField()->metadata()); + converted_fields.emplace_back(data_field.Id(), binary_field, data_field.Description()); + } + return converted_fields; +} + } // namespace paimon diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index 505cad57..c13e24f9 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -20,8 +20,10 @@ #pragma once #include +#include #include #include +#include #include "paimon/result.h" #include "paimon/visibility.h" @@ -33,6 +35,10 @@ class Schema; class StructArray; } // namespace arrow +namespace paimon { +class DataField; +} // namespace paimon + namespace paimon { /// Utils for blob type. class PAIMON_EXPORT BlobUtils { @@ -41,23 +47,30 @@ class PAIMON_EXPORT BlobUtils { ~BlobUtils() = delete; struct SeparatedSchemas { - /// Non-blob fields + /// Non-blob fields (includes inline blob fields when inline_fields is provided) std::shared_ptr main_schema; - /// Blob fields only + /// Blob fields that go to separate .blob files std::shared_ptr blob_schema; }; struct SeparatedStructArrays { - /// Non-blob fields + /// Non-blob fields (includes inline blob fields when inline_fields is provided). + /// nullptr when all fields are stored in blob files. std::shared_ptr main_array; - /// Blob fields only + /// Blob fields that go to separate .blob files std::shared_ptr blob_array; }; - static SeparatedSchemas SeparateBlobSchema(const std::shared_ptr& schema); + /// Separates schema with inline field awareness. + /// BLOB fields in inline_fields stay in main_schema; others go to blob_schema. + static SeparatedSchemas SeparateBlobSchema(const std::shared_ptr& schema, + const std::set& inline_fields); + /// Separates array with inline field awareness. + /// BLOB fields in inline_fields stay in main_array; others go to blob_array. static Result SeparateBlobArray( - const std::shared_ptr& struct_array); + const std::shared_ptr& struct_array, + const std::set& inline_fields); static bool IsBlobField(const std::shared_ptr& field); static bool IsBlobMetadata(const std::shared_ptr& metadata); @@ -66,6 +79,18 @@ class PAIMON_EXPORT BlobUtils { static std::shared_ptr ToArrowField( const std::string& field_name, bool nullable = false, std::unordered_map metadata = {}); + + static Status ValidateBlobInlineFields(const std::shared_ptr& struct_array, + const std::set& field_names, + const std::string& config_label); + + /// Converts inline blob DataFields from large_binary to binary type. + /// Inline blob fields use large_binary in the table schema (because they are BLOB type), + /// but are stored as binary in data files. This conversion aligns the field type with + /// the actual on-disk storage format for correct reading. + static std::vector ConvertBlobInlineDataFields( + const std::vector& data_fields, + const std::vector& blob_inline_fields); }; } // namespace paimon diff --git a/src/paimon/common/data/blob_utils_test.cpp b/src/paimon/common/data/blob_utils_test.cpp index d042ecda..d699d555 100644 --- a/src/paimon/common/data/blob_utils_test.cpp +++ b/src/paimon/common/data/blob_utils_test.cpp @@ -22,51 +22,59 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/types/data_field.h" #include "paimon/data/blob.h" +#include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { class BlobUtilsTest : public ::testing::Test { - private: + public: std::shared_ptr CreateBlobMetadata() { std::unordered_map blob_metadata_map = { {BlobDefs::kExtensionTypeKey, BlobDefs::kExtensionTypeValue}}; return std::make_shared(blob_metadata_map); } + + private: + std::shared_ptr pool_ = GetDefaultPool(); }; TEST_F(BlobUtilsTest, IsBlobMetadata) { auto correct_metadata = CreateBlobMetadata(); - EXPECT_TRUE(BlobUtils::IsBlobMetadata(correct_metadata)); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(nullptr)); + ASSERT_TRUE(BlobUtils::IsBlobMetadata(correct_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(nullptr)); std::unordered_map wrong_metadata_map = { {BlobDefs::kExtensionTypeKey, "paimon.type.varchar"}}; auto wrong_metadata = std::make_shared(wrong_metadata_map); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(wrong_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(wrong_metadata)); std::unordered_map no_extension_metadata_map = { {"other_key", BlobDefs::kExtensionTypeValue}}; auto no_extension_metadata = std::make_shared(no_extension_metadata_map); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(no_extension_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(no_extension_metadata)); } TEST_F(BlobUtilsTest, IsBlobField) { std::shared_ptr blob_field = BlobUtils::ToArrowField("f1", true); - EXPECT_TRUE(BlobUtils::IsBlobField(blob_field)); + ASSERT_TRUE(BlobUtils::IsBlobField(blob_field)); auto int_field = arrow::field("i_int", arrow::int32()); - EXPECT_FALSE(BlobUtils::IsBlobField(int_field)); + ASSERT_FALSE(BlobUtils::IsBlobField(int_field)); auto binary_field_no_meta = arrow::field("b_no_meta", arrow::large_binary()); - EXPECT_FALSE(BlobUtils::IsBlobField(binary_field_no_meta)); + ASSERT_FALSE(BlobUtils::IsBlobField(binary_field_no_meta)); auto wrong_meta = std::make_shared( std::unordered_map{{"other_key", "value"}}); auto binary_field_wrong_meta = arrow::field("b_wrong_meta", arrow::large_binary(), false, wrong_meta); - EXPECT_FALSE(BlobUtils::IsBlobField(binary_field_wrong_meta)); + ASSERT_FALSE(BlobUtils::IsBlobField(binary_field_wrong_meta)); } TEST_F(BlobUtilsTest, SeparateBlobSchema) { @@ -77,7 +85,8 @@ TEST_F(BlobUtilsTest, SeparateBlobSchema) { std::shared_ptr original_schema = arrow::schema({int_field, string_field, blob_field_1}); - BlobUtils::SeparatedSchemas schemas = BlobUtils::SeparateBlobSchema(original_schema); + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(original_schema, /*inline_fields=*/{}); std::shared_ptr expected_main_schema = arrow::schema({int_field, string_field}); @@ -88,17 +97,46 @@ TEST_F(BlobUtilsTest, SeparateBlobSchema) { } { std::shared_ptr no_blob_schema = arrow::schema({int_field, string_field}); - BlobUtils::SeparatedSchemas no_blob_schemas = BlobUtils::SeparateBlobSchema(no_blob_schema); + BlobUtils::SeparatedSchemas no_blob_schemas = + BlobUtils::SeparateBlobSchema(no_blob_schema, /*inline_fields=*/{}); ASSERT_TRUE(no_blob_schemas.main_schema->Equals(*no_blob_schema)); ASSERT_EQ(no_blob_schemas.blob_schema->num_fields(), 0); } { std::shared_ptr only_blob_schema = arrow::schema({blob_field_1}); BlobUtils::SeparatedSchemas only_blob_schemas = - BlobUtils::SeparateBlobSchema(only_blob_schema); + BlobUtils::SeparateBlobSchema(only_blob_schema, /*inline_fields=*/{}); ASSERT_TRUE(only_blob_schemas.blob_schema->Equals(*only_blob_schema)); ASSERT_EQ(only_blob_schemas.main_schema->num_fields(), 0); } + { + // Inline blob field stays in main_schema instead of going to blob_schema + auto blob_field_2 = BlobUtils::ToArrowField("f4_blob_2", false); + std::shared_ptr schema = + arrow::schema({int_field, blob_field_1, blob_field_2, string_field}); + + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(schema, /*inline_fields=*/{"f3_blob_1"}); + + // f3_blob_1 is inline -> stays in main; f4_blob_2 goes to blob + std::shared_ptr expected_main = + arrow::schema({int_field, blob_field_1, string_field}); + ASSERT_TRUE(schemas.main_schema->Equals(*expected_main)); + + std::shared_ptr expected_blob = arrow::schema({blob_field_2}); + ASSERT_TRUE(schemas.blob_schema->Equals(*expected_blob)); + } + { + // All blob fields are inline -> blob_schema is empty + std::shared_ptr schema = + arrow::schema({int_field, blob_field_1, string_field}); + + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(schema, /*inline_fields=*/{"f3_blob_1"}); + + ASSERT_TRUE(schemas.main_schema->Equals(*schema)); + ASSERT_EQ(schemas.blob_schema->num_fields(), 0); + } } TEST_F(BlobUtilsTest, SeparateBlobArray) { @@ -128,7 +166,8 @@ TEST_F(BlobUtilsTest, SeparateBlobArray) { std::shared_ptr struct_array = std::static_pointer_cast(raw_struct_array); - ASSERT_OK_AND_ASSIGN(auto separated, BlobUtils::SeparateBlobArray(struct_array)); + ASSERT_OK_AND_ASSIGN(auto separated, + BlobUtils::SeparateBlobArray(struct_array, /*inline_fields=*/{})); std::shared_ptr expected_main_type = arrow::struct_({int_field, string_field}); ASSERT_TRUE(separated.main_array->type()->Equals(*expected_main_type)); @@ -140,6 +179,266 @@ TEST_F(BlobUtilsTest, SeparateBlobArray) { ASSERT_TRUE(separated.blob_array->type()->Equals(*expected_blob_type)); ASSERT_EQ(separated.blob_array->num_fields(), 1); ASSERT_TRUE(separated.blob_array->field(0)->Equals(*blob_array_data)); + + // All blob fields are inline -> should return error (no blob field to separate) + ASSERT_NOK_WITH_MSG( + BlobUtils::SeparateBlobArray(struct_array, /*inline_fields=*/{"f2_blob"}), + "SeparateBlobArray expects at least one non-inline blob field, but got none."); + + // All fields are blob with no inline -> no main array is needed + auto all_blob_struct = arrow::StructArray::Make({blob_array_data}, {blob_field}).ValueOrDie(); + auto all_blob_sa = std::dynamic_pointer_cast(all_blob_struct); + ASSERT_OK_AND_ASSIGN(auto all_blob_separated, + BlobUtils::SeparateBlobArray(all_blob_sa, /*inline_fields=*/{})); + ASSERT_EQ(nullptr, all_blob_separated.main_array); + ASSERT_TRUE(all_blob_separated.blob_array->Equals(*all_blob_sa)); +} + +TEST_F(BlobUtilsTest, SeparateBlobArrayWithPartialInline) { + auto int_field = arrow::field("f1_int", arrow::int32()); + std::shared_ptr blob_field_1 = BlobUtils::ToArrowField("f2_blob_1", false); + std::shared_ptr blob_field_2 = BlobUtils::ToArrowField("f3_blob_2", true); + auto schema = arrow::schema({int_field, blob_field_1, blob_field_2}); + + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder blob_builder_1; + ASSERT_TRUE(blob_builder_1.Append("a", 1).ok()); + ASSERT_TRUE(blob_builder_1.Append("b", 1).ok()); + auto blob_array_1 = blob_builder_1.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder blob_builder_2; + ASSERT_TRUE(blob_builder_2.Append("x", 1).ok()); + ASSERT_TRUE(blob_builder_2.AppendNull().ok()); + auto blob_array_2 = blob_builder_2.Finish().ValueOrDie(); + + auto raw_struct_array = + arrow::StructArray::Make({int_array, blob_array_1, blob_array_2}, schema->fields()) + .ValueOrDie(); + auto struct_array = std::static_pointer_cast(raw_struct_array); + + // f2_blob_1 is inline, f3_blob_2 goes to blob + ASSERT_OK_AND_ASSIGN(auto separated, BlobUtils::SeparateBlobArray( + struct_array, /*inline_fields=*/{"f2_blob_1"})); + + std::shared_ptr expected_main_type = arrow::struct_({int_field, blob_field_1}); + ASSERT_TRUE(separated.main_array->type()->Equals(*expected_main_type)); + ASSERT_EQ(separated.main_array->num_fields(), 2); + ASSERT_TRUE(separated.main_array->field(0)->Equals(*int_array)); + ASSERT_TRUE(separated.main_array->field(1)->Equals(*blob_array_1)); + + std::shared_ptr expected_blob_type = arrow::struct_({blob_field_2}); + ASSERT_TRUE(separated.blob_array->type()->Equals(*expected_blob_type)); + ASSERT_EQ(separated.blob_array->num_fields(), 1); + ASSERT_TRUE(separated.blob_array->field(0)->Equals(*blob_array_2)); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsEmptyFields) { + // Empty inline_descriptor_fields -> always OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("random_data").ok()); + auto array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {}, "blob-descriptor-field")); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsFieldNotPresent) { + // Field not in struct_array -> skip, OK + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(42).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({int_array}, {arrow::field("f0", arrow::int32())}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + // "b0" does not exist in the struct -> should pass + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithValidDescriptor) { + // Valid BlobDescriptor bytes -> OK + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool_); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithNullValue) { + // Null values in blob column -> skip, OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.AppendNull().ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithRawBytes) { + // Raw bytes (not a descriptor) -> error + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("not_a_descriptor_just_raw_data").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field"), + "BLOB inline field b0 require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMixedValidAndInvalid) { + // First row is valid descriptor, second row is raw bytes -> error on row 1 + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool_); + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + ASSERT_TRUE(builder.Append("raw_bytes_not_descriptor").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field"), + "BLOB inline field b0 require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMultipleFields) { + // Two inline fields: b0 is valid, b1 has raw bytes -> error on b1 + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool_); + + arrow::LargeBinaryBuilder b0_builder; + ASSERT_TRUE(b0_builder.Append(serialized->data(), serialized->size()).ok()); + auto b0_array = b0_builder.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder b1_builder; + ASSERT_TRUE(b1_builder.Append("invalid_raw_data").ok()); + auto b1_array = b1_builder.Finish().ValueOrDie(); + + auto struct_array = + arrow::StructArray::Make({b0_array, b1_array}, + {BlobUtils::ToArrowField("b0"), BlobUtils::ToArrowField("b1")}) + .ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG( + BlobUtils::ValidateBlobInlineFields(sa, {"b0", "b1"}, "blob-descriptor-field"), + "BLOB inline field b1 require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsEmptyFields) { + // Empty view_fields -> always OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("random_data").ok()); + auto array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsFieldNotPresent) { + // Field not in struct_array -> skip, OK + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(42).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({int_array}, {arrow::field("f0", arrow::int32())}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithValidViewStruct) { + // A BlobViewStruct value is accepted for a view field. + BlobViewStruct view_struct(Identifier("db", "tbl"), /*field_id=*/2, /*row_id=*/5); + auto serialized = view_struct.Serialize(pool_); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithNullValue) { + // Null values in view column -> skip, OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.AppendNull().ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithRawBytes) { + // Raw bytes -> error + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("raw_bytes_not_view").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field"), + "BLOB inline field view require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsRejectsBlobDescriptor) { + // A BlobDescriptor value is NOT accepted for a view field. + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field"), + "BLOB inline field view require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, TestConvertBlobInlineDataFields) { + // Schema with a blob field (large_binary with blob metadata) and normal fields. + auto blob_field = BlobUtils::ToArrowField("blob_col", /*nullable=*/true); + std::vector data_fields = {DataField(0, arrow::field("int_col", arrow::int32())), + DataField(1, blob_field), + DataField(2, arrow::field("str_col", arrow::utf8()))}; + + // Without inline fields — blob_col stays as large_binary + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {}); + ASSERT_EQ(result.size(), 3); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::LARGE_BINARY); + } + + // With inline fields — blob_col should be converted from large_binary to binary + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {"blob_col"}); + ASSERT_EQ(result.size(), 3); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::BINARY); + ASSERT_EQ(result[1].Name(), "blob_col"); + ASSERT_EQ(result[1].Nullable(), true); + // Other fields unchanged + ASSERT_EQ(result[0].ArrowField()->type()->id(), arrow::Type::INT32); + ASSERT_EQ(result[2].ArrowField()->type()->id(), arrow::Type::STRING); + } + + // Non-matching inline field name — no conversion should happen + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {"non_existent_field"}); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::LARGE_BINARY); + } } } // namespace paimon::test diff --git a/src/paimon/common/data/blob_view_struct.cpp b/src/paimon/common/data/blob_view_struct.cpp new file mode 100644 index 00000000..e5bd2e9e --- /dev/null +++ b/src/paimon/common/data/blob_view_struct.cpp @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "paimon/common/data/blob_view_struct.h" + +#include + +#include "fmt/format.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/common/utils/murmurhash_utils.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/byte_order.h" +#include "paimon/io/data_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/status.h" + +namespace paimon { +PAIMON_UNIQUE_PTR BlobViewStruct::Serialize(const std::shared_ptr& pool) const { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + out.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + out.WriteValue(kCurrentVersion); + out.WriteValue(kMagic); + std::string identifier = identifier_.GetFullName(); + out.WriteValue(static_cast(identifier.size())); + auto uri_bytes = std::make_shared(identifier, pool.get()); + out.WriteBytes(uri_bytes); + out.WriteValue(field_id_); + out.WriteValue(row_id_); + return MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); +} + +Result> BlobViewStruct::Deserialize(const char* buffer, + uint64_t size) { + auto input_stream = std::make_shared(buffer, size); + DataInputStream in(std::move(input_stream)); + in.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + PAIMON_ASSIGN_OR_RAISE(int8_t version, in.ReadValue()); + if (version != kCurrentVersion) { + return Status::Invalid(fmt::format( + "Expecting BlobViewStruct version to be {}, but found {}.", kCurrentVersion, version)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + if (kMagic != magic) { + return Status::Invalid( + fmt::format("Invalid BlobViewStruct: missing magic header. Expected magic: {}, " + "but found {}", + kMagic, magic)); + } + PAIMON_ASSIGN_OR_RAISE(int32_t length, in.ReadValue()); + std::string identifier_str(length, '\0'); + PAIMON_RETURN_NOT_OK(in.Read(identifier_str.data(), identifier_str.size())); + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, Identifier::FromString(identifier_str)); + return std::make_unique(identifier, field_id, row_id); +} + +Result BlobViewStruct::IsBlobViewStruct(const char* buffer, uint64_t size) { + if (size < kMinViewLength) { + return false; + } + auto input_stream = std::make_shared(buffer, size); + DataInputStream in(std::move(input_stream)); + in.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + PAIMON_ASSIGN_OR_RAISE(int8_t version, in.ReadValue()); + if (version != kCurrentVersion) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + return kMagic == magic; +} + +std::string BlobViewStruct::ToString() const { + return fmt::format("BlobViewStruct{{identifier={}, fieldId={}, rowId={}}}", + identifier_.GetFullName(), field_id_, row_id_); +} + +bool BlobViewStruct::operator==(const BlobViewStruct& other) const { + if (this == &other) { + return true; + } + return field_id_ == other.field_id_ && row_id_ == other.row_id_ && + identifier_ == other.identifier_; +} + +bool BlobViewStruct::operator!=(const BlobViewStruct& other) const { + return !(*this == other); +} + +int32_t BlobViewStruct::HashCode() const { + int32_t hash = + MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&field_id_), + /*offset=*/0, sizeof(field_id_), identifier_.HashCode()); + return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&row_id_), + /*offset=*/0, sizeof(row_id_), hash); +} +} // namespace paimon diff --git a/src/paimon/common/data/blob_view_struct.h b/src/paimon/common/data/blob_view_struct.h new file mode 100644 index 00000000..971a80b3 --- /dev/null +++ b/src/paimon/common/data/blob_view_struct.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +/// Serialized metadata for a BLOB view field. +/// A blob view only stores the coordinates needed to locate the original blob value in the +/// upstream table: identifier, field_id and row_id. The actual blob data is +/// resolved at read time by scanning the upstream table. +class BlobViewStruct { + public: + BlobViewStruct(const Identifier& identifier, int32_t field_id, int64_t row_id) + : identifier_(identifier), field_id_(field_id), row_id_(row_id) {} + + const Identifier& GetIdentifier() const { + return identifier_; + } + + int32_t FieldId() const { + return field_id_; + } + + int64_t RowId() const { + return row_id_; + } + + static Result> Deserialize(const char* buffer, uint64_t size); + static Result IsBlobViewStruct(const char* buffer, uint64_t size); + PAIMON_UNIQUE_PTR Serialize(const std::shared_ptr& pool) const; + std::string ToString() const; + int32_t HashCode() const; + + bool operator==(const BlobViewStruct& other) const; + bool operator!=(const BlobViewStruct& other) const; + + private: + static constexpr int64_t kMagic = 0x424C4F4256494557l; + static constexpr int8_t kCurrentVersion = 1; + /// one byte for version, eight bytes for magic number. + static constexpr uint64_t kMinViewLength = 9; + + Identifier identifier_; + int32_t field_id_; + int64_t row_id_; +}; + +/// Resolves a BlobViewStruct into the serialized BlobDescriptor bytes stored in the upstream +/// table. Returns nullptr when the referenced source-table cell is null. +using BlobViewResolver = std::function>(const BlobViewStruct&)>; + +} // namespace paimon + +namespace std { +template <> +struct hash { + size_t operator()(const paimon::BlobViewStruct& blob_view_struct) const { + return blob_view_struct.HashCode(); + } +}; + +} // namespace std diff --git a/src/paimon/common/data/blob_view_struct_test.cpp b/src/paimon/common/data/blob_view_struct_test.cpp new file mode 100644 index 00000000..1e7bf4ea --- /dev/null +++ b/src/paimon/common/data/blob_view_struct_test.cpp @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "paimon/common/data/blob_view_struct.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class BlobViewStructTest : public testing::Test { + public: + std::shared_ptr pool_ = GetDefaultPool(); + Identifier identifier_ = Identifier("test_db", "test_table"); + BlobViewStruct view_struct_ = BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1024); +}; + +TEST_F(BlobViewStructTest, TestConstructorAndGetters) { + ASSERT_EQ(view_struct_.GetIdentifier().GetDatabaseName(), "test_db"); + ASSERT_EQ(view_struct_.GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(view_struct_.FieldId(), 7); + ASSERT_EQ(view_struct_.RowId(), 1024); +} + +TEST_F(BlobViewStructTest, TestSerializeDeserializeRoundTrip) { + auto serialized = view_struct_.Serialize(pool_); + ASSERT_NE(serialized, nullptr); + ASSERT_GT(serialized->size(), 0u); + + ASSERT_OK_AND_ASSIGN(auto restored, + BlobViewStruct::Deserialize(serialized->data(), serialized->size())); + ASSERT_EQ(restored->GetIdentifier().GetDatabaseName(), "test_db"); + ASSERT_EQ(restored->GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(restored->FieldId(), 7); + ASSERT_EQ(restored->RowId(), 1024); +} + +TEST_F(BlobViewStructTest, TestDeserializeWithInvalidVersion) { + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[0] = '\x02'; // invalid version (current is 1). + ASSERT_NOK_WITH_MSG(BlobViewStruct::Deserialize(serialized->data(), serialized->size()), + "Expecting BlobViewStruct version to be 1, but found 2"); +} + +TEST_F(BlobViewStructTest, TestDeserializeWithInvalidMagic) { + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[1] = '\x00'; + ASSERT_NOK_WITH_MSG(BlobViewStruct::Deserialize(serialized->data(), serialized->size()), + "missing magic header"); +} + +TEST_F(BlobViewStructTest, TestToString) { + std::string debug_str = view_struct_.ToString(); + ASSERT_EQ(debug_str, "BlobViewStruct{identifier=test_db.test_table, fieldId=7, rowId=1024}"); +} + +TEST_F(BlobViewStructTest, TestEqual) { + { + // test equal itself + ASSERT_EQ(view_struct_, view_struct_); + } + { + // test equal + BlobViewStruct other_view_struct = + BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1024); + ASSERT_EQ(view_struct_, other_view_struct); + } + { + // test wrong identifier + Identifier wrong_identifier = Identifier("db", "table"); + BlobViewStruct wrong_view_struct = + BlobViewStruct(wrong_identifier, /*field_id=*/7, /*row_id=*/1024); + ASSERT_NE(view_struct_, wrong_view_struct); + } + { + // test wrong field_id + BlobViewStruct wrong_view_struct = + BlobViewStruct(identifier_, /*field_id=*/8, /*row_id=*/1024); + ASSERT_NE(view_struct_, wrong_view_struct); + } + { + // test wrong row_id + BlobViewStruct wrong_view_struct = + BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1000); + ASSERT_NE(view_struct_, wrong_view_struct); + } +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructValid) { + auto serialized = view_struct_.Serialize(pool_); + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_TRUE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithTooShortBuffer) { + // Buffer shorter than 9 bytes should return false + std::vector short_buffer = {0x02, 0x43, 0x53, 0x45, 0x44, 0x42, 0x4F, 0x4C}; + ASSERT_OK_AND_ASSIGN( + bool result, BlobViewStruct::IsBlobViewStruct(short_buffer.data(), short_buffer.size())); + ASSERT_FALSE(result); + + // Empty buffer + ASSERT_OK_AND_ASSIGN(bool empty_result, BlobViewStruct::IsBlobViewStruct(nullptr, 0)); + ASSERT_FALSE(empty_result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithFutureVersion) { + // Version > CURRENT_VERSION should return false (not an error) + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[0] = '\x02'; // set version to 2 (> CURRENT_VERSION) + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_FALSE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithWrongMagic) { + // Wrong magic number should return false + auto serialized = view_struct_.Serialize(pool_); + // Corrupt the magic bytes (bytes 1-8) + (*serialized)[1] = '\x00'; + (*serialized)[2] = '\x00'; + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_FALSE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithRandomData) { + // Random data that doesn't match format + std::vector random_data = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}; + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(random_data.data(), random_data.size())); + ASSERT_FALSE(result); +} +} // namespace paimon::test diff --git a/src/paimon/common/data/columnar/columnar_batch_context.h b/src/paimon/common/data/columnar/columnar_batch_context.h index 2d35c0dd..adbe5416 100644 --- a/src/paimon/common/data/columnar/columnar_batch_context.h +++ b/src/paimon/common/data/columnar/columnar_batch_context.h @@ -17,10 +17,13 @@ */ #pragma once +#include #include +#include #include #include "arrow/array/array_base.h" +#include "arrow/util/bit_util.h" namespace arrow { class StructArray; @@ -29,12 +32,106 @@ class StructArray; namespace paimon { class MemoryPool; +/// Pre-cached column metadata for fast access without virtual function calls or checked_cast. +struct CachedColumnMeta { + /// Null bitmap pointer. nullptr means no nulls (all valid). + const uint8_t* null_bitmap = nullptr; + /// Arrow array offset (for sliced arrays). + int64_t array_offset = 0; + /// For fixed-width types: raw pointer to values buffer (buffer[1]). + /// For variable-length types (STRING/BINARY): raw pointer to data buffer (buffer[2]). + const uint8_t* values_data = nullptr; + /// For variable-length types (STRING/BINARY): raw pointer to offsets buffer (buffer[1]). + const int32_t* offsets = nullptr; + + /// Fast null check: directly reads validity bitmap bit. + inline bool IsNull(int64_t row_id) const { + return null_bitmap != nullptr && + !arrow::bit_util::GetBit(null_bitmap, array_offset + row_id); + } + + /// Fast fixed-width value access (INT8/INT16/INT32/INT64/FLOAT/DOUBLE/DATE32/TIMESTAMP). + template + inline T GetFixed(int64_t row_id) const { + return reinterpret_cast(values_data)[array_offset + row_id]; + } + + /// Fast boolean value access (bit-packed in values buffer). + inline bool GetBool(int64_t row_id) const { + return arrow::bit_util::GetBit(values_data, array_offset + row_id); + } + + /// Fast string_view access for non-dictionary STRING/BINARY columns. + inline std::string_view GetVarLenView(int64_t row_id) const { + int64_t idx = array_offset + row_id; + int32_t start = offsets[idx]; + int32_t length = offsets[idx + 1] - start; + return {reinterpret_cast(values_data) + start, static_cast(length)}; + } +}; + struct ColumnarBatchContext { ColumnarBatchContext(const arrow::ArrayVector& array_vec_in, const std::shared_ptr& pool_in) - : pool(pool_in), array_vec(array_vec_in) {} + : pool(pool_in), array_vec(array_vec_in) { + BuildCachedMeta(); + } std::shared_ptr pool; arrow::ArrayVector array_vec; + /// Pre-cached metadata per column for fast access. + std::vector cached_meta; + + private: + void BuildCachedMeta() { + cached_meta.resize(array_vec.size()); + for (size_t i = 0; i < array_vec.size(); i++) { + const auto* array = array_vec[i].get(); + auto& meta = cached_meta[i]; + meta.array_offset = array->offset(); + + // Cache null bitmap + if (array->null_count() != 0) { + meta.null_bitmap = array->null_bitmap_data(); + } + + // Cache data pointers based on type + const auto& array_data = array->data(); + switch (array->type_id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::DATE32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: { + // Fixed-width: values in buffer[1] + if (array_data->buffers.size() > 1 && array_data->buffers[1]) { + meta.values_data = array_data->buffers[1]->data(); + } + break; + } + case arrow::Type::STRING: + case arrow::Type::BINARY: { + // Variable-length: offsets in buffer[1], data in buffer[2] + if (array_data->buffers.size() > 2) { + if (array_data->buffers[1]) { + meta.offsets = + reinterpret_cast(array_data->buffers[1]->data()); + } + if (array_data->buffers[2]) { + meta.values_data = array_data->buffers[2]->data(); + } + } + break; + } + default: + // TIMESTAMP, DECIMAL, DICTIONARY, LIST, MAP, STRUCT — not cached, use array_vec + // fallback + break; + } + } + } }; } // namespace paimon diff --git a/src/paimon/common/data/columnar/columnar_row_ref.h b/src/paimon/common/data/columnar/columnar_row_ref.h index b08fdf6e..45582c26 100644 --- a/src/paimon/common/data/columnar/columnar_row_ref.h +++ b/src/paimon/common/data/columnar/columnar_row_ref.h @@ -38,10 +38,11 @@ namespace paimon { class Bytes; /// Columnar row view which shares batch-level context to reduce per-row overhead. +/// Uses pre-cached column metadata for fast field access without virtual function calls. class ColumnarRowRef : public InternalRow { public: ColumnarRowRef(std::shared_ptr ctx, int64_t row_id) - : ctx_(std::move(ctx)), row_id_(row_id) {} + : ctx_(std::move(ctx)), cached_meta_ptr_(ctx_->cached_meta.data()), row_id_(row_id) {} Result GetRowKind() const override { return row_kind_; @@ -56,47 +57,39 @@ class ColumnarRowRef : public InternalRow { } bool IsNullAt(int32_t pos) const override { - return ctx_->array_vec[pos]->IsNull(row_id_); + return cached_meta_ptr_[pos].IsNull(row_id_); } bool GetBoolean(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetBool(row_id_); } char GetByte(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return static_cast(cached_meta_ptr_[pos].GetFixed(row_id_)); } int16_t GetShort(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int32_t GetInt(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int32_t GetDate(int32_t pos) const override { - return ColumnarUtils::GetGenericValue( - ctx_->array_vec[pos].get(), row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int64_t GetLong(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } float GetFloat(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } double GetDouble(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } BinaryString GetString(int32_t pos) const override { @@ -106,6 +99,11 @@ class ColumnarRowRef : public InternalRow { } std::string_view GetStringView(int32_t pos) const override { + auto& meta = cached_meta_ptr_[pos]; + if (meta.values_data && meta.offsets) { + return meta.GetVarLenView(row_id_); + } + // Fallback for dictionary-encoded or uncached types return ColumnarUtils::GetView(ctx_->array_vec[pos].get(), row_id_); } @@ -130,6 +128,7 @@ class ColumnarRowRef : public InternalRow { private: std::shared_ptr ctx_; + const CachedColumnMeta* cached_meta_ptr_; const RowKind* row_kind_ = RowKind::Insert(); int64_t row_id_; }; diff --git a/src/paimon/common/data/columnar/columnar_utils.h b/src/paimon/common/data/columnar/columnar_utils.h index ca9f58ef..c1270ed5 100644 --- a/src/paimon/common/data/columnar/columnar_utils.h +++ b/src/paimon/common/data/columnar/columnar_utils.h @@ -109,7 +109,9 @@ class ColumnarUtils { MemoryPool* pool) { auto view = GetView(array, pos); std::shared_ptr bytes = Bytes::AllocateBytes(view.size(), pool); - memcpy(bytes->data(), view.data(), view.size()); + if (!view.empty()) { + memcpy(bytes->data(), view.data(), view.size()); + } return bytes; } }; diff --git a/src/paimon/common/data/decimal.cpp b/src/paimon/common/data/decimal.cpp index 0ce5b220..20bf4664 100644 --- a/src/paimon/common/data/decimal.cpp +++ b/src/paimon/common/data/decimal.cpp @@ -110,10 +110,14 @@ Decimal Decimal::FromUnscaledBytes(int32_t precision, int32_t scale, Bytes* byte int32_t Decimal::clz_u128(uint128_t u) { uint64_t hi = u >> 64; + if (hi != 0) { + return __builtin_clzll(hi); + } uint64_t lo = u; - int32_t retval[3] = {__builtin_clzll(hi), __builtin_clzll(lo) + 64, 128}; - int32_t idx = !hi + ((!lo) & (!hi)); - return retval[idx]; + if (lo != 0) { + return __builtin_clzll(lo) + 64; + } + return 128; } int32_t Decimal::count_leading_zero_bytes(uint128_t u) { diff --git a/src/paimon/common/data/decimal_test.cpp b/src/paimon/common/data/decimal_test.cpp index 038b3d3b..f8931b10 100644 --- a/src/paimon/common/data/decimal_test.cpp +++ b/src/paimon/common/data/decimal_test.cpp @@ -100,7 +100,7 @@ TEST(DecimalTest, TestCompatibleWithJava) { auto pool = GetDefaultPool(); auto file_system = std::make_unique(); auto file_name = paimon::test::GetDataDir() + "/decimal_bytes.data"; - uint64_t file_length = file_system->GetFileStatus(file_name).value()->GetLen(); + int64_t file_length = file_system->GetFileStatus(file_name).value()->GetLen(); ASSERT_GT(file_length, 0); ASSERT_OK_AND_ASSIGN(auto input_stream, file_system->Open(file_name)); auto data_bytes = Bytes::AllocateBytes(file_length, pool.get()); @@ -133,7 +133,12 @@ TEST(DecimalTest, TestCompatibleWithJava) { } TEST(DecimalTest, TestCompareTo) { - auto CheckResult = [](const Decimal& decimal1, const Decimal& decimal2) { + auto min_int128 = DecimalUtils::StrToInt128("-170141183460469231731687303715884105728").value(); + auto can_negate = [min_int128](const Decimal& decimal) { + return decimal.Value() != min_int128; + }; + + auto CheckResult = [can_negate](const Decimal& decimal1, const Decimal& decimal2) { ASSERT_FALSE(decimal1 < decimal1); ASSERT_FALSE(decimal1 > decimal1); ASSERT_EQ(decimal1, decimal1); @@ -146,6 +151,9 @@ TEST(DecimalTest, TestCompareTo) { ASSERT_EQ(decimal3.CompareTo(decimal1), 0); ASSERT_EQ(decimal3, decimal1); + if (!can_negate(decimal1) || !can_negate(decimal2)) { + return; + } Decimal negative_decimal1(decimal1.Precision(), decimal1.Scale(), -decimal1.Value()); Decimal negative_decimal2(decimal2.Precision(), decimal2.Scale(), -decimal2.Value()); ASSERT_EQ(negative_decimal1.CompareTo(negative_decimal2), 1); @@ -157,10 +165,13 @@ TEST(DecimalTest, TestCompareTo) { ASSERT_EQ(negative_decimal3, negative_decimal1); }; - auto CheckEqual = [](const Decimal& decimal1, const Decimal& decimal2) { + auto CheckEqual = [can_negate](const Decimal& decimal1, const Decimal& decimal2) { ASSERT_EQ(decimal1.CompareTo(decimal2), 0); ASSERT_EQ(decimal2.CompareTo(decimal1), 0); + if (!can_negate(decimal1) || !can_negate(decimal2)) { + return; + } Decimal negative_decimal1(decimal1.Precision(), decimal1.Scale(), -decimal1.Value()); Decimal negative_decimal2(decimal2.Precision(), decimal2.Scale(), -decimal2.Value()); ASSERT_EQ(negative_decimal1.CompareTo(negative_decimal2), 0); diff --git a/src/paimon/common/data/generic_row.h b/src/paimon/common/data/generic_row.h index 7e5da05d..36c3f471 100644 --- a/src/paimon/common/data/generic_row.h +++ b/src/paimon/common/data/generic_row.h @@ -223,6 +223,15 @@ class GenericRow : public InternalRow { return row; } + void ResetFields() { + for (auto& field : fields_) { + field = VariantType{}; + } + kind_ = RowKind::Insert(); + row_holder_.clear(); + bytes_holder_.reset(); + } + private: /// The array to store the actual internal format values. std::vector fields_; diff --git a/src/paimon/common/data/generic_row_test.cpp b/src/paimon/common/data/generic_row_test.cpp index 3d6abdee..1aca20a7 100644 --- a/src/paimon/common/data/generic_row_test.cpp +++ b/src/paimon/common/data/generic_row_test.cpp @@ -107,4 +107,35 @@ TEST(GenericRowTest, TestSimple) { "00:00:00.100000020,123.45678998765432145678,array,row,map,null)", row.ToString()); } + +TEST(GenericRowTest, TestResetFields) { + auto pool = GetDefaultPool(); + GenericRow row(3); + row.SetField(0, static_cast(1)); + row.SetField(1, BinaryString::FromString("old", pool.get())); + row.SetField(2, Bytes::AllocateBytes("bytes", pool.get())); + row.SetRowKind(RowKind::Delete()); + + ASSERT_FALSE(row.IsNullAt(0)); + ASSERT_FALSE(row.IsNullAt(1)); + ASSERT_FALSE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::Delete()); + + row.ResetFields(); + + ASSERT_EQ(row.GetFieldCount(), 3); + ASSERT_TRUE(row.IsNullAt(0)); + ASSERT_TRUE(row.IsNullAt(1)); + ASSERT_TRUE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::Insert()); + + row.SetField(0, static_cast(2)); + row.SetField(1, BinaryString::FromString("new", pool.get())); + row.SetRowKind(RowKind::UpdateAfter()); + + ASSERT_EQ(row.GetInt(0), static_cast(2)); + ASSERT_EQ(row.GetString(1), BinaryString::FromString("new", pool.get())); + ASSERT_TRUE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::UpdateAfter()); +} } // namespace paimon::test diff --git a/src/paimon/common/data/internal_row.cpp b/src/paimon/common/data/internal_row.cpp index 1744e15b..abc54e7c 100644 --- a/src/paimon/common/data/internal_row.cpp +++ b/src/paimon/common/data/internal_row.cpp @@ -113,7 +113,7 @@ Result InternalRow::CreateFieldGetter( }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/data/serializer/binary_row_serializer.cpp b/src/paimon/common/data/serializer/binary_row_serializer.cpp index 826f7558..56075ac0 100644 --- a/src/paimon/common/data/serializer/binary_row_serializer.cpp +++ b/src/paimon/common/data/serializer/binary_row_serializer.cpp @@ -26,9 +26,9 @@ #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" namespace paimon { diff --git a/src/paimon/common/data/serializer/binary_row_serializer_test.cpp b/src/paimon/common/data/serializer/binary_row_serializer_test.cpp index 22c2f6b8..51a6d3aa 100644 --- a/src/paimon/common/data/serializer/binary_row_serializer_test.cpp +++ b/src/paimon/common/data/serializer/binary_row_serializer_test.cpp @@ -25,12 +25,12 @@ #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/binary_string.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { diff --git a/src/paimon/common/data/serializer/binary_serializer_utils.cpp b/src/paimon/common/data/serializer/binary_serializer_utils.cpp index 4675e54e..544e0ca2 100644 --- a/src/paimon/common/data/serializer/binary_serializer_utils.cpp +++ b/src/paimon/common/data/serializer/binary_serializer_utils.cpp @@ -106,7 +106,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptrSetNullAt(pos, type_id); return Status::OK(); - } else if (type_id != arrow::Type::type::DECIMAL && + } else if (type_id != arrow::Type::type::DECIMAL128 && type_id != arrow::Type::type::TIMESTAMP) { // if row writer, exclude decimal and timestamp when set null writer->SetNullAt(pos); @@ -170,7 +170,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptr(type.get()); assert(decimal_type); auto precision = decimal_type->precision(); diff --git a/src/paimon/common/data/serializer/row_compacted_serializer.cpp b/src/paimon/common/data/serializer/row_compacted_serializer.cpp index 4bf247b3..491f3a65 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer.cpp +++ b/src/paimon/common/data/serializer/row_compacted_serializer.cpp @@ -97,7 +97,7 @@ Result RowCompactedSerializer::CompareField(const FieldInfo& field_info PAIMON_ASSIGN_OR_RAISE(Timestamp val2, reader2->ReadTimestamp(field_info.precision)); return val1 == val2 ? 0 : (val1 < val2 ? -1 : 1); } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { PAIMON_ASSIGN_OR_RAISE(Decimal val1, reader1->ReadDecimal(field_info.precision, field_info.scale)); PAIMON_ASSIGN_OR_RAISE(Decimal val2, @@ -127,7 +127,7 @@ Result RowCompactedSerializer::CreateSliceComparat arrow::internal::checked_pointer_cast(field_type); assert(timestamp_type); field_infos[i].precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); - } else if (field_type->id() == arrow::Type::type::DECIMAL) { + } else if (field_type->id() == arrow::Type::type::DECIMAL128) { auto decimal_type = arrow::internal::checked_pointer_cast(field_type); assert(decimal_type); @@ -278,7 +278,7 @@ Result RowCompactedSerializer::CreateFieldR }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); @@ -413,7 +413,7 @@ Result RowCompactedSerializer::CreateFieldW }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/data/serializer/row_compacted_serializer.h b/src/paimon/common/data/serializer/row_compacted_serializer.h index 9acda1cb..6d73f606 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer.h +++ b/src/paimon/common/data/serializer/row_compacted_serializer.h @@ -23,10 +23,10 @@ #include "arrow/api.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_writer.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/common/utils/var_length_int_utils.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class RowCompactedSerializer { diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp new file mode 100644 index 00000000..ae5228ed --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" + +#include +#include + +namespace paimon { + +LruMapSharedShreddingColumnAllocator::LruMapSharedShreddingColumnAllocator(int32_t num_columns) + : MapSharedShreddingColumnAllocator(num_columns), + col_field_(num_columns, -1), + last_used_(num_columns, 0) {} + +int32_t LruMapSharedShreddingColumnAllocator::SelectColumn( + const std::vector& candidates, + const std::vector& planned_col_to_field) const { + int32_t selected_col = candidates.front(); + int64_t selected_last_used = std::numeric_limits::max(); + for (int32_t col : candidates) { + if (planned_col_to_field[col] == -1) { + return col; + } + if (last_used_[col] < selected_last_used) { + selected_col = col; + selected_last_used = last_used_[col]; + } + } + return selected_col; +} + +void LruMapSharedShreddingColumnAllocator::UpdateLastUsed(const RowAllocation& allocation) { + bool touched = false; + for (int32_t col = 0; col < num_columns_; ++col) { + if (allocation.col_to_field[col] != -1) { + last_used_[col] = lru_clock_; + touched = true; + } + } + if (touched) { + ++lru_clock_; + } +} + +RowAllocation LruMapSharedShreddingColumnAllocator::AllocateRow( + const std::vector& field_ids) { + std::vector sorted_field_ids = field_ids; + std::sort(sorted_field_ids.begin(), sorted_field_ids.end()); + + RowAllocation allocation; + allocation.col_to_field.assign(num_columns_, -1); + std::vector next_col_to_field = col_field_; + std::vector used_cols(num_columns_, false); + std::vector unassigned; + + for (int32_t field_id : sorted_field_ids) { + auto it = std::find(col_field_.begin(), col_field_.end(), field_id); + if (it != col_field_.end()) { + int32_t col = static_cast(it - col_field_.begin()); + used_cols[col] = true; + allocation.col_to_field[col] = field_id; + } else { + unassigned.push_back(field_id); + } + } + + for (int32_t field_id : unassigned) { + std::vector candidates; + candidates.reserve(num_columns_); + for (int32_t col = 0; col < num_columns_; ++col) { + if (!used_cols[col]) { + candidates.push_back(col); + } + } + + if (candidates.empty()) { + allocation.overflow_fields.push_back(field_id); + continue; + } + + int32_t col = SelectColumn(candidates, next_col_to_field); + used_cols[col] = true; + allocation.col_to_field[col] = field_id; + next_col_to_field[col] = field_id; + } + + UpdateLastUsed(allocation); + col_field_ = next_col_to_field; + CommitRow(allocation, sorted_field_ids); + return allocation; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..8c2bb1a4 --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that keeps current column assignments across rows and evicts least-recently-used +/// columns. +/// +/// Keys that are still assigned to physical columns keep those columns when they appear again. +/// New keys, including keys that were previously evicted, use empty columns first; if no empty +/// column is available, the allocator replaces the least-recently-used physical column. +class LruMapSharedShreddingColumnAllocator : public MapSharedShreddingColumnAllocator { + public: + /// @param num_columns Number of available physical columns. + explicit LruMapSharedShreddingColumnAllocator(int32_t num_columns); + + RowAllocation AllocateRow(const std::vector& field_ids) override; + + private: + int32_t SelectColumn(const std::vector& candidates, + const std::vector& planned_col_to_field) const; + void UpdateLastUsed(const RowAllocation& allocation); + + int64_t lru_clock_ = 0; + std::vector col_field_; + std::vector last_used_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..a04f886e --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(LruMapSharedShreddingColumnAllocatorTest, AllocatesWithHitRetainEvictAndOverflow) { + LruMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({0, 1, 2}); + ExpectAllocation(row0, {0, 1, 2}, {}); + + RowAllocation row1 = allocator.AllocateRow({0, 1}); + ExpectAllocation(row1, {0, 1, -1}, {}); + + RowAllocation row2 = allocator.AllocateRow({3, 4, 5}); + ExpectAllocation(row2, {4, 5, 3}, {}); + + RowAllocation row3 = allocator.AllocateRow({0, 3, 4, 5}); + ExpectAllocation(row3, {4, 5, 3}, {0}); + + ASSERT_EQ(4, allocator.GetMaxRowWidth()); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{0}), field_to_columns.at(0)); + ASSERT_EQ((std::set{1}), field_to_columns.at(1)); + ASSERT_EQ((std::set{2}), field_to_columns.at(2)); + ASSERT_EQ((std::set{2}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{1}), field_to_columns.at(5)); + ASSERT_EQ((std::set{0}), allocator.GetOverflowFieldSet()); +} + +TEST(LruMapSharedShreddingColumnAllocatorTest, HandlesEmptyRows) { + LruMapSharedShreddingColumnAllocator allocator(2); + + RowAllocation empty_row = allocator.AllocateRow({}); + ExpectAllocation(empty_row, {-1, -1}, {}); + ASSERT_EQ(0, allocator.GetMaxRowWidth()); + + RowAllocation row = allocator.AllocateRow({7}); + ExpectAllocation(row, {7, -1}, {}); + ASSERT_EQ(1, allocator.GetMaxRowWidth()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp new file mode 100644 index 00000000..2a9b69dc --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/builder.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" +#include "paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/core_options.h" +namespace paimon { +/// Checks that a dynamic_cast result is not null, returning Status::Invalid on failure. +#define PAIMON_CHECK_NOT_NULL(ptr, msg) \ + do { \ + if (PAIMON_UNLIKELY((ptr) == nullptr)) { \ + return Status::Invalid(msg); \ + } \ + } while (false) + +namespace { + +Result> CreateMapSharedShreddingColumnAllocator( + int32_t num_columns, MapSharedShreddingColumnPlacementPolicy placement_policy) { + switch (placement_policy) { + case MapSharedShreddingColumnPlacementPolicy::PLAIN: + return std::make_unique(num_columns); + case MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL: + return std::make_unique(num_columns); + case MapSharedShreddingColumnPlacementPolicy::LRU: + return std::make_unique(num_columns); + } + return Status::Invalid("unknown shared-shredding column placement policy"); +} + +} // namespace + +Result> MapSharedShreddingBatchConverter::Create( + const std::shared_ptr& logical_schema, + const std::shared_ptr& context, const CoreOptions& options, + const std::shared_ptr& pool) { + std::map field_to_num_columns = context->ComputeNextK(); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns)); + std::vector contexts; + std::vector shredding_field_names; + contexts.reserve(field_to_num_columns.size()); + shredding_field_names.reserve(field_to_num_columns.size()); + // Iterate in schema field order (not map order) so that shredding_field_names_ + // matches the order in which shredding columns appear in the schema. + // This is critical for the sequential matching logic in Convert(). + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + const std::string& name = logical_schema->field(i)->name(); + auto it = field_to_num_columns.find(name); + if (it != field_to_num_columns.end()) { + int32_t num_columns = it->second; + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingColumnPlacementPolicy placement_policy, + options.GetMapSharedShreddingColumnPlacementPolicy(name)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr allocator, + CreateMapSharedShreddingColumnAllocator(num_columns, placement_policy)); + contexts.emplace_back(name, num_columns, std::move(allocator)); + shredding_field_names.push_back(name); + } + } + return std::shared_ptr( + new MapSharedShreddingBatchConverter(logical_schema, physical_schema, std::move(contexts), + std::move(shredding_field_names), pool)); +} + +MapSharedShreddingBatchConverter::MapSharedShreddingBatchConverter( + const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, std::vector&& contexts, + std::vector&& shredding_field_names, const std::shared_ptr& pool) + : logical_schema_(logical_schema), + physical_schema_(physical_schema), + contexts_(std::move(contexts)), + shredding_field_names_(std::move(shredding_field_names)), + pool_(GetArrowPool(pool)) {} + +const std::shared_ptr& MapSharedShreddingBatchConverter::GetPhysicalSchema() const { + return physical_schema_; +} + +Result> MapSharedShreddingBatchConverter::Convert( + ArrowArray* logical_batch) { + std::shared_ptr logical_type = arrow::struct_(logical_schema_->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_array, + arrow::ImportArray(logical_batch, logical_type)); + auto logical_struct = std::dynamic_pointer_cast(logical_array); + PAIMON_CHECK_NOT_NULL(logical_struct, + "MapSharedShreddingBatchConverter: input is not a StructArray"); + + int32_t num_fields = logical_schema_->num_fields(); + arrow::ArrayVector physical_columns; + physical_columns.reserve(num_fields); + size_t context_idx = 0; + for (int32_t col = 0; col < num_fields; ++col) { + auto column = logical_struct->field(col); + const std::string& field_name = logical_schema_->field(col)->name(); + if (context_idx < shredding_field_names_.size() && + shredding_field_names_[context_idx] == field_name) { + auto physical_struct_type = physical_schema_->field(col)->type(); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr physical_column, + ConvertOneColumn(column, physical_struct_type, &contexts_[context_idx])); + physical_columns.push_back(std::move(physical_column)); + ++context_idx; + } else { + physical_columns.push_back(column); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr physical_struct, + arrow::StructArray::Make(physical_columns, physical_schema_->field_names())); + + std::unique_ptr result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*physical_struct, result.get())); + return result; +} + +Result> MapSharedShreddingBatchConverter::ConvertOneColumn( + const std::shared_ptr& map_column, + const std::shared_ptr& physical_struct_type, ColumnContext* context) const { + auto map_array = std::dynamic_pointer_cast(map_column); + PAIMON_CHECK_NOT_NULL(map_array, "MapSharedShreddingBatchConverter: column is not a MapArray"); + + int64_t num_rows = map_array->length(); + int32_t num_cols = context->num_columns; + + auto keys_array = std::dynamic_pointer_cast(map_array->keys()); + PAIMON_CHECK_NOT_NULL(keys_array, + "MapSharedShreddingBatchConverter: MAP keys are not StringArray"); + auto values_array = map_array->items(); + + // Create StructBuilder from physical struct type — it owns all child builders. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr struct_builder_base, + arrow::MakeBuilder(physical_struct_type, pool_.get())); + auto* struct_builder = dynamic_cast(struct_builder_base.get()); + PAIMON_CHECK_NOT_NULL(struct_builder, + "MapSharedShreddingBatchConverter: failed to create StructBuilder"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Reserve(num_rows)); + + // Extract child builders: [field_mapping, col_0..K-1, overflow] + auto* field_mapping_builder = + dynamic_cast(struct_builder->field_builder(0)); + PAIMON_CHECK_NOT_NULL(field_mapping_builder, + "MapSharedShreddingBatchConverter: field_mapping is not a ListBuilder"); + auto* field_mapping_value_builder = + dynamic_cast(field_mapping_builder->value_builder()); + PAIMON_CHECK_NOT_NULL( + field_mapping_value_builder, + "MapSharedShreddingBatchConverter: field_mapping value is not Int32Builder"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_mapping_builder->Reserve(num_rows)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_mapping_value_builder->Reserve(num_rows * num_cols)); + + std::vector col_builders_raw; + col_builders_raw.reserve(num_cols); + for (int32_t c = 0; c < num_cols; ++c) { + arrow::ArrayBuilder* col_builder = struct_builder->field_builder(1 + c); + PAIMON_CHECK_NOT_NULL(col_builder, "MapSharedShreddingBatchConverter: col builder is null"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(col_builder->Reserve(num_rows)); + col_builders_raw.push_back(col_builder); + } + + int32_t overflow_field_idx = 1 + num_cols; + auto* overflow_builder = + dynamic_cast(struct_builder->field_builder(overflow_field_idx)); + PAIMON_CHECK_NOT_NULL(overflow_builder, + "MapSharedShreddingBatchConverter: overflow is not a MapBuilder"); + auto* overflow_key_builder = + dynamic_cast(overflow_builder->key_builder()); + PAIMON_CHECK_NOT_NULL(overflow_key_builder, + "MapSharedShreddingBatchConverter: overflow key is not Int32Builder"); + arrow::ArrayBuilder* overflow_value_builder = overflow_builder->item_builder(); + PAIMON_CHECK_NOT_NULL(overflow_value_builder, + "MapSharedShreddingBatchConverter: overflow value builder is null"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->Reserve(num_rows)); + + // Process each row + for (int64_t row = 0; row < num_rows; ++row) { + if (map_array->IsNull(row)) { + // StructBuilder::AppendNull() auto-appends empty values to all children. + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + + int64_t start = map_array->value_offset(row); + int64_t length = map_array->value_length(row); + + // Extract field ids and build lookup map + std::vector field_ids; + std::unordered_map field_id_to_value_index; + ExtractRowFields(keys_array, start, length, &context->dict, &field_ids, + &field_id_to_value_index); + + // Allocate columns + RowAllocation allocation = context->allocator->AllocateRow(field_ids); + + // Fill sub-columns + PAIMON_RETURN_NOT_OK(AppendFieldMapping(allocation, num_cols, field_mapping_builder, + field_mapping_value_builder)); + PAIMON_RETURN_NOT_OK(AppendColumnValues(values_array, allocation, field_id_to_value_index, + num_cols, col_builders_raw)); + PAIMON_RETURN_NOT_OK(AppendOverflow(values_array, allocation, field_id_to_value_index, + overflow_builder, overflow_key_builder, + overflow_value_builder)); + } + + // Finalize + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Finish(&result)); + return result; +} + +void MapSharedShreddingBatchConverter::ExtractRowFields( + const std::shared_ptr& keys_array, int64_t start, int64_t length, + MapSharedShreddingFieldDict* dict, std::vector* field_ids_out, + std::unordered_map* field_id_to_value_index_out) const { + field_ids_out->clear(); + field_ids_out->reserve(length); + field_id_to_value_index_out->clear(); + field_id_to_value_index_out->reserve(length); + for (int64_t j = 0; j < length; ++j) { + std::string key_str = keys_array->GetString(start + j); + int32_t field_id = dict->GetOrAssign(key_str); + field_ids_out->push_back(field_id); + (*field_id_to_value_index_out)[field_id] = start + j; + } +} + +Status MapSharedShreddingBatchConverter::AppendFieldMapping( + const RowAllocation& allocation, int32_t num_cols, arrow::ListBuilder* list_builder, + arrow::Int32Builder* value_builder) const { + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Append()); + for (int32_t c = 0; c < num_cols; ++c) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(allocation.col_to_field[c])); + } + return Status::OK(); +} + +Status MapSharedShreddingBatchConverter::AppendColumnValues( + const std::shared_ptr& values_array, const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, int32_t num_cols, + const std::vector& col_builders) const { + for (int32_t c = 0; c < num_cols; ++c) { + int32_t assigned_field_id = allocation.col_to_field[c]; + if (assigned_field_id == -1) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(col_builders[c]->AppendNull()); + } else { + auto it = field_id_to_value_index.find(assigned_field_id); + if (PAIMON_UNLIKELY(it == field_id_to_value_index.end())) { + return Status::Invalid( + fmt::format("MapSharedShreddingBatchConverter: field_id {} assigned to col {} " + "but not found in current row", + assigned_field_id, c)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + col_builders[c]->AppendArraySlice(*values_array->data(), it->second, 1)); + } + } + return Status::OK(); +} + +Status MapSharedShreddingBatchConverter::AppendOverflow( + const std::shared_ptr& values_array, const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + arrow::MapBuilder* overflow_builder, arrow::Int32Builder* overflow_key_builder, + arrow::ArrayBuilder* overflow_value_builder) const { + if (allocation.overflow_fields.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->AppendNull()); + return Status::OK(); + } + + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->Append()); + for (int32_t overflow_field_id : allocation.overflow_fields) { + auto it = field_id_to_value_index.find(overflow_field_id); + if (PAIMON_UNLIKELY(it == field_id_to_value_index.end())) { + return Status::Invalid(fmt::format( + "MapSharedShreddingBatchConverter: overflow field_id {} not found in current row", + overflow_field_id)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_key_builder->Append(overflow_field_id)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + overflow_value_builder->AppendArraySlice(*values_array->data(), it->second, 1)); + } + return Status::OK(); +} + +Result MapSharedShreddingBatchConverter::BuildFieldMeta( + const std::string& field_name) const { + for (const auto& context : contexts_) { + if (context.field_name == field_name) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = context.dict.GetNameToId(); + // Convert set -> vector for field_to_columns + for (const auto& [field_id, col_set] : context.allocator->GetFieldToColumns()) { + meta.field_to_columns[field_id] = + std::vector(col_set.begin(), col_set.end()); + } + meta.overflow_field_set = context.allocator->GetOverflowFieldSet(); + meta.num_columns = context.num_columns; + meta.max_row_width = context.allocator->GetMaxRowWidth(); + return meta; + } + } + return Status::Invalid(fmt::format( + "cannot find field_name '{}' in MapSharedShreddingBatchConverter contexts", field_name)); +} + +Result MapSharedShreddingBatchConverter::GetMaxRowWidth( + const std::string& field_name) const { + for (const auto& context : contexts_) { + if (context.field_name == field_name) { + return context.allocator->GetMaxRowWidth(); + } + } + return Status::Invalid(fmt::format( + "cannot find field_name '{}' in MapSharedShreddingBatchConverter contexts", field_name)); +} + +const std::vector& MapSharedShreddingBatchConverter::GetShreddingColumnNames() const { + return shredding_field_names_; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h new file mode 100644 index 00000000..b808692c --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/type_fwd.h" +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" +#include "paimon/common/data/shredding/map_shared_shredding_field_dict.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/data/shredding/shredding_batch_converter.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowArray; + +namespace paimon { + +class CoreOptions; +class MapSharedShreddingContext; + +/// Converts logical batches containing MAP columns into physical batches +/// where each shared-shredding MAP column is replaced by +/// STRUCT<__field_mapping, __col_0..K-1, __overflow>. +/// +/// Non-shared-shredding columns are passed through unchanged. +/// Each shared-shredding column has its own FieldDict and ColumnAllocator. +class MapSharedShreddingBatchConverter : public ShreddingBatchConverter { + public: + /// Creates a converter for one file write cycle. + /// Computes per-file K from context, builds physical schema, and constructs the converter. + /// @param logical_schema The original schema with MAP columns. + /// @param context The cross-file shared context for K adaptation. + /// @param options CoreOptions used to read each column's placement policy. + /// @param pool Paimon memory pool for Arrow allocations. + /// @return The converter. + static Result> Create( + const std::shared_ptr& logical_schema, + const std::shared_ptr& context, const CoreOptions& options, + const std::shared_ptr& pool); + + /// Returns the physical schema produced for this converter. + const std::shared_ptr& GetPhysicalSchema() const override; + + /// Converts a logical batch to a physical batch. + /// @param logical_batch Input ArrowArray (C ABI) with logical schema. Consumed on success. + /// @return Owned physical ArrowArray (C ABI) with physical schema. + Result> Convert(ArrowArray* logical_batch) override; + + /// Builds MapSharedShreddingFieldMeta for one shredding column (by field name). + /// Called at file close to serialize metadata. + Result BuildFieldMeta(const std::string& field_name) const; + + /// Returns the maximum number of entries observed in one row for a shredding column. + Result GetMaxRowWidth(const std::string& field_name) const; + + /// Returns all shredding column field names. + const std::vector& GetShreddingColumnNames() const; + + private: + /// Per-column context for one shared-shredding MAP column. + struct ColumnContext { + std::string field_name; + int32_t num_columns; // K + MapSharedShreddingFieldDict dict; + std::unique_ptr allocator; + + ColumnContext(const std::string& field_name, int32_t num_columns, + std::unique_ptr&& allocator) + : field_name(field_name), num_columns(num_columns), allocator(std::move(allocator)) {} + }; + + /// Constructs a converter. + /// @param logical_schema The original schema with MAP columns. + /// @param physical_schema The physical schema (MAP columns replaced with STRUCT). + /// @param contexts Per-shredding-column conversion contexts. + /// @param shredding_field_names Shared-shredding field names in schema order. + /// @param pool Paimon memory pool for Arrow allocations. + MapSharedShreddingBatchConverter(const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, + std::vector&& contexts, + std::vector&& shredding_field_names, + const std::shared_ptr& pool); + + /// Converts one MAP column to physical STRUCT for all rows. + /// @param physical_struct_type The physical struct type from physical_schema for this column. + Result> ConvertOneColumn( + const std::shared_ptr& map_column, + const std::shared_ptr& physical_struct_type, ColumnContext* context) const; + + /// Extracts field ids and builds field_id -> value_index map for one row. + void ExtractRowFields(const std::shared_ptr& keys_array, int64_t start, + int64_t length, MapSharedShreddingFieldDict* dict, + std::vector* field_ids_out, + std::unordered_map* field_id_to_value_index_out) const; + + /// Appends __field_mapping list for one row. + Status AppendFieldMapping(const RowAllocation& allocation, int32_t num_cols, + arrow::ListBuilder* list_builder, + arrow::Int32Builder* value_builder) const; + + /// Appends __col_0..K-1 values for one row. + Status AppendColumnValues(const std::shared_ptr& values_array, + const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + int32_t num_cols, + const std::vector& col_builders) const; + + /// Appends __overflow entries for one row. + Status AppendOverflow(const std::shared_ptr& values_array, + const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + arrow::MapBuilder* overflow_builder, + arrow::Int32Builder* overflow_key_builder, + arrow::ArrayBuilder* overflow_value_builder) const; + + std::shared_ptr logical_schema_; + std::shared_ptr physical_schema_; + std::vector contexts_; + std::vector shredding_field_names_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp new file mode 100644 index 00000000..df99c021 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp @@ -0,0 +1,632 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/core/core_options.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon { + +using arrow::ipc::internal::json::ArrayFromJSON; + +class MapSharedShreddingBatchConverterTest : public ::testing::Test { + protected: + std::shared_ptr pool_ = GetDefaultPool(); + + /// Builds a logical struct array from JSON, converts it, and returns the physical result. + std::shared_ptr RunConvert(const std::shared_ptr& logical_type, + const std::string& input_json, + const std::shared_ptr& physical_type, + MapSharedShreddingBatchConverter* converter) { + auto input = ArrayFromJSON(logical_type, input_json).ValueOrDie(); + ArrowArray c_input; + EXPECT_TRUE(arrow::ExportArray(*input, &c_input).ok()); + EXPECT_OK_AND_ASSIGN(auto c_output, converter->Convert(&c_input)); + return arrow::ImportArray(c_output.get(), physical_type).ValueOrDie(); + } + + /// Asserts that two arrays are equal, printing both on failure. + void AssertArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) { + ASSERT_TRUE(expected->Equals(*actual)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); + } + + Result MakeCoreOptions(const std::map& field_to_policy) { + std::map options; + for (const auto& [field_name, policy] : field_to_policy) { + options.emplace( + "fields." + field_name + ".map.shared-shredding.column-placement-policy", policy); + } + return CoreOptions::FromMap(options); + } +}; + +TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { + // Schema: id(INT32), tags(MAP), K=3 + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Input: 2 rows — [id, tags] + // Row0: id=100, tags={a:1, b:2} + // Row1: id=200, tags={b:3, c:4, a:5} + auto actual = RunConvert(logical_type, R"([ + [100, [["a", 1], ["b", 2]]], + [200, [["b", 3], ["c", 4], ["a", 5]]] + ])", + physical_type, converter.get()); + // Expected physical: [id, [mapping, col0, col1, col2, overflow]] + // Row0: a=fid0->col0, b=fid1->col1, col2 unused + // Row1: b=fid1->col0, c=fid2->col1, a=fid0->col2 + auto expected = ArrayFromJSON(physical_type, R"([ + [100, [[0, 1, -1], 1, 2, null, null]], + [200, [[1, 2, 0], 3, 4, 5, null]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"tags"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2, K=3, max_row_width=3, no overflow + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 2}}, {1, {0, 1}}, {2, {1}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, MapWithNullValue) { + auto logical_schema = + arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))}); + auto context = + std::make_shared(std::map{{"metrics", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields()); + + auto actual = + RunConvert(logical_type, R"([[[["a", null], ["b", 20]]]])", physical_type, converter.get()); + auto expected = ArrayFromJSON(physical_type, R"([[[[0, 1], null, 20, null]]])").ValueOrDie(); + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 2; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, OverflowWhenExceedK) { + auto logical_schema = + arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))}); + auto context = + std::make_shared(std::map{{"metrics", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields()); + + auto actual = RunConvert(logical_type, R"([[[["a", 10], ["b", 20], ["c", 30]]]])", + physical_type, converter.get()); + auto expected = ArrayFromJSON(physical_type, R"([[[[0, 1], 10, 20, [[2, 30]]]]])").ValueOrDie(); + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, EmptyAndNullMaps) { + auto logical_schema = + arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))}); + auto context = + std::make_shared(std::map{{"metrics", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields()); + + auto actual = RunConvert(logical_type, R"([[null], [[]]])", physical_type, converter.get()); + auto expected = + ArrayFromJSON(physical_type, R"([[null], [[[-1, -1], null, null, null]]])").ValueOrDie(); + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 0; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { + // MAP>, K=2 + auto value_type = arrow::struct_({ + arrow::field("x", arrow::int32()), + arrow::field("y", arrow::float64()), + }); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("props", arrow::map(arrow::utf8(), value_type)), + }); + auto context = + std::make_shared(std::map{{"props", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"props", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: props={a:[1,1.5], b:[null,2.5]} → a=fid0->col0, b=fid1->col1; b.x is null + // Row1: props={c:[3,3.5]} → c=fid2->col0 + // Row2: props={a:[null,null], c:[5,5.5], b:[6,6.5]} → 3 fields K=2: overflow b; a has all-null + // struct Row3: props=null → null row + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [1, 1.5]], ["b", [null, 2.5]]]], + [2, [["c", [3, 3.5]]]], + [3, [["a", [null, null]], ["c", [5, 5.5]], ["b", [6, 6.5]]]], + [4, null] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [1, 1.5], [null, 2.5], null]], + [2, [[2, -1], [3, 3.5], null, null]], + [3, [[0, 2], [null, null], [5, 5.5], [[1, [6, 6.5]]]]], + [4, null] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"props"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, b overflowed in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 1}}}; + expected_meta.overflow_field_set = {1}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("props").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { + // MAP>, K=2 + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), + }); + auto context = + std::make_shared(std::map{{"tags", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: tags={a:[1,null,2], b:[3]} → a=fid0->col0, b=fid1->col1; a has null element + // Row1: tags={a:[null]} → a=fid0->col0; single null element list + // Row2: tags={c:[5,6,7]} → c=fid2->col0 + // Row3: tags={b:[8], a:[9,10], c:[null]} → 3 fields K=2: overflow c; c has null element + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [1, null, 2]], ["b", [3]]]], + [2, [["a", [null]]]], + [3, [["c", [5, 6, 7]]]], + [4, [["b", [8]], ["a", [9, 10]], ["c", [null]]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [1, null, 2], [3], null]], + [2, [[0, -1], [null], null, null]], + [3, [[2, -1], [5, 6, 7], null, null]], + [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"tags"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {0, 1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { + // MAP>, K=2 + auto inner_map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("nested", arrow::map(arrow::utf8(), inner_map_type)), + }); + auto context = + std::make_shared(std::map{{"nested", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"nested", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: nested={a:{x:1,y:null}, b:{z:3}} → a=fid0->col0, b=fid1->col1; a has null value + // Row1: nested={c:{p:null}} → c=fid2->col0; inner map value all null + // Row2: nested=null → null row + // Row3: nested={a:{m:7}, b:{n:8}, c:{o:9}} → 3 fields K=2: overflow c + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [["x", 1], ["y", null]]], ["b", [["z", 3]]]]], + [2, [["c", [["p", null]]]]], + [3, null], + [4, [["a", [["m", 7]]], ["b", [["n", 8]]], ["c", [["o", 9]]]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [["x", 1], ["y", null]], [["z", 3]], null]], + [2, [[2, -1], [["p", null]], null, null]], + [3, null], + [4, [[0, 1], [["m", 7]], [["n", 8]], [[2, [["o", 9]]]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"nested"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("nested").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { + // MAP, meta:MAP>>, K=2 + auto value_type = arrow::struct_({ + arrow::field("score", arrow::int32()), + arrow::field("tags", arrow::list(arrow::utf8())), + arrow::field("meta", arrow::map(arrow::utf8(), arrow::int32())), + }); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("data", arrow::map(arrow::utf8(), value_type)), + }); + auto context = + std::make_shared(std::map{{"data", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"data", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: a=[10,["t1","t2"],{x:1}], b=[20,["t3"],{y:2,z:3}] → a=fid0->col0, b=fid1->col1 + // Row1: c=[null,null,{p:null}] → c=fid2->col0; nulls inside + // struct Row2: a=[30,[null,"t4"],{}], b=[null,[],{q:5}], c=[40,["t5"],{r:6}] → overflow c Row3: + // null → null row + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [10, ["t1", "t2"], [["x", 1]]]], ["b", [20, ["t3"], [["y", 2], ["z", 3]]]]]], + [2, [["c", [null, null, [["p", null]]]]]], + [3, [["a", [30, [null, "t4"], []]], ["b", [null, [], [["q", 5]]]], ["c", [40, ["t5"], [["r", 6]]]]]], + [4, null] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [10, ["t1", "t2"], [["x", 1]]], [20, ["t3"], [["y", 2], ["z", 3]]], null]], + [2, [[2, -1], [null, null, [["p", null]]], null, null]], + [3, [[0, 1], [30, [null, "t4"], []], [null, [], [["q", 5]]], [[2, [40, ["t5"], [["r", 6]]]]]]], + [4, null] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"data"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("data").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { + // Schema: id(INT32), tags(MAP) K=2, attrs(MAP) K=3 + // Tests that two shared-shredding MAP columns in the same schema are independently converted. + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::float64())), + }); + auto context = std::make_shared( + std::map{{"tags", 2}, {"attrs", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeCoreOptions({{"tags", "plain"}, {"attrs", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: id=1, tags={a:10, b:20}, attrs={x:1.1, y:2.2} + // tags: a=fid0->col0, b=fid1->col1 (fits K=2) + // attrs: x=fid0->col0, y=fid1->col1, col2 unused (fits K=3) + // Row1: id=2, tags={c:30, a:40, b:50}, attrs={z:3.3} + // tags: c=fid2->col0, a=fid0->col1; b overflows (K=2) + // attrs: z=fid2->col0, col1/col2 unused + // Row2: id=3, tags=null, attrs={x:4.4, y:5.5, z:6.6, w:7.7} + // tags: null + // attrs: x=fid0->col0, y=fid1->col1, z=fid2->col2; w overflows (K=3) + auto actual = RunConvert(logical_type, R"([ + [1, [["a", 10], ["b", 20]], [["x", 1.1], ["y", 2.2]]], + [2, [["c", 30], ["a", 40], ["b", 50]], [["z", 3.3]]], + [3, null, [["x", 4.4], ["y", 5.5], ["z", 6.6], ["w", 7.7]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], 10, 20, null], [[0, 1, -1], 1.1, 2.2, null, null]], + [2, [[2, 0], 30, 40, [[1, 50]]], [[2, -1, -1], 3.3, null, null, null]], + [3, null, [[0, 1, 2], 4.4, 5.5, 6.6, [[3, 7.7]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames returns both columns in order + ASSERT_EQ(std::vector({"tags", "attrs"}), converter->GetShreddingColumnNames()); + + // Verify BuildFieldMeta for tags: a=0,b=1,c=2; K=2, max_row_width=3 + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + tags_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}}; + tags_meta.overflow_field_set = {1}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 3; + ASSERT_EQ(tags_meta, converter->BuildFieldMeta("tags").value()); + + // Verify BuildFieldMeta for attrs: x=0,y=1,z=2,w=3; K=3, max_row_width=4 + MapSharedShreddingFieldMeta attrs_meta; + attrs_meta.name_to_id = {{"x", 0}, {"y", 1}, {"z", 2}, {"w", 3}}; + attrs_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 2}}}; + attrs_meta.overflow_field_set = {3}; + attrs_meta.num_columns = 3; + attrs_meta.max_row_width = 4; + ASSERT_EQ(attrs_meta, converter->BuildFieldMeta("attrs").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, BuildFieldMetaInvalidFieldName) { + // Schema: id(INT32), tags(MAP), K=3 + // Only "tags" is a shredding field + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + + // Valid case: "tags" exists + ASSERT_OK_AND_ASSIGN([[maybe_unused]] auto meta, converter->BuildFieldMeta("tags")); + ASSERT_OK_AND_ASSIGN(int32_t max_row_width, converter->GetMaxRowWidth("tags")); + ASSERT_EQ(0, max_row_width); + + // Invalid case: "id" is not a shredding field + ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("id"), "cannot find field_name 'id'"); + ASSERT_NOK_WITH_MSG(converter->GetMaxRowWidth("id"), "cannot find field_name 'id'"); + + // Invalid case: nonexistent field name + ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("nonexistent"), + "cannot find field_name 'nonexistent'"); + ASSERT_NOK_WITH_MSG(converter->GetMaxRowWidth("nonexistent"), + "cannot find field_name 'nonexistent'"); +} + +TEST_F(MapSharedShreddingBatchConverterTest, SequentialPlacementUsesSmallestColumn) { + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "sequential"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + auto actual = RunConvert(logical_type, R"([ + [100, [["a", 1], ["b", 2]]], + [200, [["b", 3], ["c", 4], ["a", 5]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [100, [[0, 1, -1], 1, 2, null, null]], + [200, [[0, 1, 2], 5, 3, 4, null]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, LruPlacementPreservesResidentColumns) { + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "lru"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + auto actual = RunConvert(logical_type, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 40], ["b", 50]]], + [3, [["d", 60], ["e", 70], ["f", 80]]], + [4, [["a", 90], ["d", 100], ["e", 110], ["f", 120]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1, 2], 10, 20, 30, null]], + [2, [[0, 1, -1], 40, 50, null, null]], + [3, [[4, 5, 3], 70, 80, 60, null]], + [4, [[4, 5, 3], 110, 120, 100, [[0, 90]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}, {"f", 5}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {2}}, {4, {0}}, {5, {1}}}; + expected_meta.overflow_field_set = {0}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 4; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, FactoryUsesMaxColumnCountForFirstFile) { + auto logical_schema = + arrow::schema({arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))}); + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "4"}})); + ASSERT_OK_AND_ASSIGN( + auto factory, MapSharedShreddingWritePlanFactory::Create(options, logical_schema, pool_)); + + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_FALSE(factory->ShouldInferWritePlan()); + ASSERT_EQ(0, factory->InferBufferRowCount()); + ASSERT_OK_AND_ASSIGN(auto converter, factory->CreateConverter("parquet", {})); + ASSERT_OK_AND_ASSIGN(auto expected, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 4}})); + ASSERT_TRUE(converter->GetPhysicalSchema()->Equals(*expected)); +} + +TEST_F(MapSharedShreddingBatchConverterTest, FactoryUsesCompletedFileStatsForNextFile) { + auto logical_schema = + arrow::schema({arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))}); + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "8"}})); + ASSERT_OK_AND_ASSIGN( + auto factory, MapSharedShreddingWritePlanFactory::Create(options, logical_schema, pool_)); + ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", {})); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto first_physical_type = arrow::struct_(first->GetPhysicalSchema()->fields()); + auto input = ArrayFromJSON(logical_type, R"([ + [[["a", 1], ["b", 2]]], + [[["c", 3], ["d", 4], ["e", 5]]] + ])") + .ValueOrDie(); + ArrowArray c_input; + ASSERT_TRUE(arrow::ExportArray(*input, &c_input).ok()); + ASSERT_OK_AND_ASSIGN(auto c_output, first->Convert(&c_input)); + auto first_output = arrow::ImportArray(c_output.get(), first_physical_type).ValueOrDie(); + ASSERT_EQ(2, first_output->length()); + ASSERT_OK(factory->OnFileCompleted(first)); + + ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", {})); + ASSERT_OK_AND_ASSIGN(auto expected_first, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 8}})); + ASSERT_OK_AND_ASSIGN(auto expected_second, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 3}})); + ASSERT_TRUE(first->GetPhysicalSchema()->Equals(*expected_first)); + ASSERT_TRUE(second->GetPhysicalSchema()->Equals(*expected_second)); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp new file mode 100644 index 00000000..42b7b467 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +#include + +namespace paimon { + +MapSharedShreddingColumnAllocator::MapSharedShreddingColumnAllocator(int32_t num_columns) + : num_columns_(num_columns) {} + +void MapSharedShreddingColumnAllocator::CommitRow(const RowAllocation& allocation, + const std::vector& field_ids) { + max_row_width_ = std::max(max_row_width_, static_cast(field_ids.size())); + + for (int32_t col = 0; col < num_columns_; ++col) { + int32_t field_id = allocation.col_to_field[col]; + if (field_id != -1) { + field_to_columns_[field_id].insert(col); + } + } + + for (int32_t field_id : allocation.overflow_fields) { + overflow_field_set_.insert(field_id); + } +} + +const std::map>& MapSharedShreddingColumnAllocator::GetFieldToColumns() + const { + return field_to_columns_; +} + +const std::set& MapSharedShreddingColumnAllocator::GetOverflowFieldSet() const { + return overflow_field_set_; +} + +int32_t MapSharedShreddingColumnAllocator::GetMaxRowWidth() const { + return max_row_width_; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..f9c18bd9 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { + +/// Per-row allocation result produced by MapSharedShreddingColumnAllocator. +struct RowAllocation { + /// Physical column assignments: col_to_field[col_index] = field_id. + /// Length is always K. Unused columns have value -1. + std::vector col_to_field; + + /// Field ids that overflowed (more fields than K physical columns). + std::vector overflow_fields; +}; + +/// Allocates shared-shredding MAP field ids to K physical columns on a per-row basis, +/// and accumulates field-level metadata (field_to_columns, overflow_field_set, max_row_width). +class MapSharedShreddingColumnAllocator { + public: + virtual ~MapSharedShreddingColumnAllocator() = default; + + /// Allocates physical columns for one row's field ids. + /// @param field_ids Field ids present in this row. + /// @return Allocation result with column assignments and overflow list. + virtual RowAllocation AllocateRow(const std::vector& field_ids) = 0; + + /// Returns accumulated field_id -> set of column indices (for MapSharedShreddingFileMeta). + const std::map>& GetFieldToColumns() const; + + /// Returns accumulated overflow field id set (for MapSharedShreddingFileMeta). + const std::set& GetOverflowFieldSet() const; + + /// Returns the maximum row width observed so far. + int32_t GetMaxRowWidth() const; + + protected: + /// @param num_columns Number of physical columns K for this shared-shredding MAP column. + explicit MapSharedShreddingColumnAllocator(int32_t num_columns); + + /// Commits a planned row allocation and updates accumulated metadata. + /// @param allocation Allocation materialized for the current row. + /// @param field_ids Field ids after allocator-specific preparation. + void CommitRow(const RowAllocation& allocation, const std::vector& field_ids); + + int32_t num_columns_; + + private: + // ---- Accumulated field-level metadata ---- + std::map> field_to_columns_; + std::set overflow_field_set_; + int32_t max_row_width_ = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp new file mode 100644 index 00000000..e5174284 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_context.h" + +#include +#include + +namespace paimon { + +MapSharedShreddingContext::MapSharedShreddingContext( + const std::map& column_to_k_max) + : column_to_k_max_(column_to_k_max) {} + +std::map MapSharedShreddingContext::ComputeNextK() const { + std::map result; + for (const auto& [field_name, k_max] : column_to_k_max_) { + auto it = recent_max_row_widths_.find(field_name); + if (it == recent_max_row_widths_.end() || it->second.empty()) { + // First file — no history, use K_max. + result[field_name] = k_max; + } else { + int32_t adaptive_width = ComputeAdaptiveWidth(it->second); + result[field_name] = std::max(1, std::min(adaptive_width, k_max)); + } + } + return result; +} + +void MapSharedShreddingContext::ReportFileStats(const std::string& field_name, + int32_t max_row_width) { + auto& window = recent_max_row_widths_[field_name]; + window.push_back(max_row_width); + if (static_cast(window.size()) > kWindowSize) { + window.erase(window.begin()); + } +} + +std::vector MapSharedShreddingContext::GetShreddingColumnNames() const { + std::vector names; + names.reserve(column_to_k_max_.size()); + for (const auto& [field_name, _] : column_to_k_max_) { + names.push_back(field_name); + } + return names; +} + +int32_t MapSharedShreddingContext::ComputeAdaptiveWidth(const std::vector& values) { + if (values.empty()) { + return 0; + } + + std::vector sorted_values(values.begin(), values.end()); + std::sort(sorted_values.begin(), sorted_values.end()); + + int32_t max_width = sorted_values.back(); + auto percentile_rank = static_cast(std::ceil(kPercentileRatio * sorted_values.size())); + percentile_rank = std::clamp(percentile_rank, 1, sorted_values.size()); + int32_t percentile_width = sorted_values[percentile_rank - 1]; + + // Use P90 to ignore far outliers, but keep max when it is close enough to normal rows. + auto relative_close_threshold = static_cast( + std::ceil(static_cast(percentile_width) * kMaxCloseRelativeRatio)); + if (max_width - percentile_width <= kMaxCloseAbsoluteSlack || + max_width <= relative_close_threshold) { + return max_width; + } + return percentile_width; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.h b/src/paimon/common/data/shredding/map_shared_shredding_context.h new file mode 100644 index 00000000..1f530a36 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.h @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { + +/// Cross-file shared context for shared-shredding MAP columns. +/// +/// Lifetime: same as one rolling writer / write-plan factory. +/// Holds per-column K_max and a sliding window of recent max_row_width +/// values to support adaptive K sizing across files. +/// +/// - First file: K = K_max (no history). +/// - Subsequent files: K = min(adaptive_width(recent_max_row_widths), K_max). +class MapSharedShreddingContext { + public: + /// @param column_to_k_max Map from field name to its K_max (from options). + explicit MapSharedShreddingContext(const std::map& column_to_k_max); + + /// Returns the K to use for each shared-shredding column in the next file. + /// First file returns K_max for all columns; subsequent files adapt + /// based on recent max_row_width observations. + std::map ComputeNextK() const; + + /// Reports the max row width observed in a completed file, for K adaptation. + /// @param field_name Field name of the shared-shredding MAP column. + /// @param max_row_width The maximum number of MAP keys in any single row of this file. + void ReportFileStats(const std::string& field_name, int32_t max_row_width); + + /// Returns the set of shared-shredding field names. + std::vector GetShreddingColumnNames() const; + + private: + static constexpr int32_t kWindowSize = 20; + static constexpr double kPercentileRatio = 0.90; + static constexpr int32_t kMaxCloseAbsoluteSlack = 4; + static constexpr double kMaxCloseRelativeRatio = 1.25; + + static int32_t ComputeAdaptiveWidth(const std::vector& values); + + /// K_max per shared-shredding field, from options. + std::map column_to_k_max_; + /// Sliding window of recent max_row_width per field, for K adaptation. + std::map> recent_max_row_widths_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp new file mode 100644 index 00000000..8f3ea276 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_context.h" + +#include +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(MapSharedShreddingContextTest, FirstFileUsesKMax) { + // No history — ComputeNextK should return K_max for every column. + std::map field_to_k_max = {{"tags", 8}, {"metrics", 4}}; + MapSharedShreddingContext context(field_to_k_max); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(2, next_k.size()); + ASSERT_EQ(8, next_k.at("tags")); + ASSERT_EQ(4, next_k.at("metrics")); +} + +TEST(MapSharedShreddingContextTest, AdaptKAfterOneFile) { + // After reporting stats from one file, K should adapt to + // min(adaptive width, K_max). + std::map field_to_k_max = {{"m", 10}}; + MapSharedShreddingContext context(field_to_k_max); + + // First file uses K_max=10. + auto k1 = context.ComputeNextK(); + ASSERT_EQ(10, k1.at("m")); + + // Report: file had max_row_width=3 for field "m". + context.ReportFileStats("m", 3); + + // Second file: K = min(3, 10) = 3. + auto k2 = context.ComputeNextK(); + ASSERT_EQ(3, k2.at("m")); +} + +TEST(MapSharedShreddingContextTest, AdaptKCappedByKMax) { + // Even if max_row_width > K_max, K should be capped at K_max. + std::map field_to_k_max = {{"m", 5}}; + MapSharedShreddingContext context(field_to_k_max); + + context.ReportFileStats("m", 100); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(5, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWhenSamplesAreClose) { + // Small sample windows still use max because max and P90 are close. + std::map field_to_k_max = {{"m", 20}}; + MapSharedShreddingContext context(field_to_k_max); + + context.ReportFileStats("m", 3); + context.ReportFileStats("m", 7); + context.ReportFileStats("m", 5); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(7, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90IgnoresSingleFarOutlier) { + std::map field_to_k_max = {{"m", 2000}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 3); + } + context.ReportFileStats("m", 1000); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(3, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWithinAbsoluteSlack) { + std::map field_to_k_max = {{"m", 20}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 3); + } + context.ReportFileStats("m", 7); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(7, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWithinRelativeSlack) { + std::map field_to_k_max = {{"m", 200}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 100); + } + context.ReportFileStats("m", 125); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(125, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90IgnoresMaxBeyondBothSlacks) { + std::map field_to_k_max = {{"m", 200}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 100); + } + context.ReportFileStats("m", 130); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(100, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, MultipleColumnsIndependent) { + // Each field adapts independently. + std::map field_to_k_max = {{"tags", 10}, {"attrs", 6}}; + MapSharedShreddingContext context(field_to_k_max); + + // First file. + auto k1 = context.ComputeNextK(); + ASSERT_EQ(10, k1.at("tags")); + ASSERT_EQ(6, k1.at("attrs")); + + // Report: tags had width 4, attrs had width 2. + context.ReportFileStats("tags", 4); + context.ReportFileStats("attrs", 2); + + auto k2 = context.ComputeNextK(); + ASSERT_EQ(4, k2.at("tags")); + ASSERT_EQ(2, k2.at("attrs")); + + // Report: tags had width 8, attrs had width 6. + context.ReportFileStats("tags", 8); + context.ReportFileStats("attrs", 6); + + auto k3 = context.ComputeNextK(); + // tags and attrs are close sample windows, so adaptive width keeps max. + ASSERT_EQ(8, k3.at("tags")); + ASSERT_EQ(6, k3.at("attrs")); +} + +TEST(MapSharedShreddingContextTest, GetShreddingColumnNames) { + std::map field_to_k_max = {{"tags", 8}, {"metrics", 4}, {"props", 16}}; + MapSharedShreddingContext context(field_to_k_max); + + auto names = context.GetShreddingColumnNames(); + ASSERT_EQ(names, std::vector({"metrics", "props", "tags"})); +} + +TEST(MapSharedShreddingContextTest, SlidingWindowEvictsOldEntries) { + // The window size is 20. After filling 20 entries, adding one more + // should evict the oldest. Verify that the evicted value no longer + // affects ComputeNextK. + std::map field_to_k_max = {{"m", 256}}; + MapSharedShreddingContext context(field_to_k_max); + + // Insert a large value as the first entry. + context.ReportFileStats("m", 200); + + // Fill the remaining 19 slots with small values. + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 3); + } + + // Window = [200, 3, 3, ..., 3] (20 entries). P90 = 3, max is a far outlier. + auto k_before = context.ComputeNextK(); + ASSERT_EQ(3, k_before.at("m")); + + // Push one more — evicts the 200. + context.ReportFileStats("m", 5); + + // Window = [3, 3, ..., 3, 5] (20 entries). Max is within the absolute slack. + auto k_after = context.ComputeNextK(); + ASSERT_EQ(5, k_after.at("m")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h b/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h new file mode 100644 index 00000000..d6c6e4be --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { + +/// File-level field name <-> field_id dictionary for shared-shredding MAP. +/// Assigns monotonically increasing ids to new field names within one file. +class MapSharedShreddingFieldDict { + public: + MapSharedShreddingFieldDict() = default; + + /// Looks up or assigns a field_id for the given field name. + /// New names get the next available id. + int32_t GetOrAssign(const std::string& name) { + auto iterator = name_to_id_.find(name); + if (iterator != name_to_id_.end()) { + return iterator->second; + } + int32_t field_id = next_id_++; + name_to_id_[name] = field_id; + return field_id; + } + + /// Returns the complete name -> field_id dictionary. + /// Used to populate MapSharedShreddingFieldMeta::name_to_id at file close. + const std::map& GetNameToId() const { + return name_to_id_; + } + + /// Returns the number of distinct field names seen so far. + int32_t Size() const { + return next_id_; + } + + private: + std::map name_to_id_; + int32_t next_id_ = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp new file mode 100644 index 00000000..65c14051 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_field_dict.h" + +#include "gtest/gtest.h" + +namespace paimon { + +TEST(MapSharedShreddingFieldDictTest, AssignMonotonicallyIncreasingIds) { + MapSharedShreddingFieldDict dict; + ASSERT_EQ(0, dict.GetOrAssign("cpu_usage")); + ASSERT_EQ(1, dict.GetOrAssign("mem_load")); + ASSERT_EQ(2, dict.GetOrAssign("disk_io")); + ASSERT_EQ(3, dict.Size()); +} + +TEST(MapSharedShreddingFieldDictTest, LookupReturnsExistingId) { + MapSharedShreddingFieldDict dict; + int32_t id = dict.GetOrAssign("alpha"); + ASSERT_EQ(id, dict.GetOrAssign("alpha")); + ASSERT_EQ(1, dict.Size()); +} + +TEST(MapSharedShreddingFieldDictTest, GetNameToId) { + MapSharedShreddingFieldDict dict; + dict.GetOrAssign("b_field"); + dict.GetOrAssign("a_field"); + + auto name_to_id = dict.GetNameToId(); + ASSERT_EQ(2u, name_to_id.size()); + ASSERT_EQ(1, name_to_id.at("a_field")); + ASSERT_EQ(0, name_to_id.at("b_field")); +} + +TEST(MapSharedShreddingFieldDictTest, EmptyDict) { + MapSharedShreddingFieldDict dict; + ASSERT_EQ(0, dict.Size()); + ASSERT_TRUE(dict.GetNameToId().empty()); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp new file mode 100644 index 00000000..dcceddc7 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" + +#include +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/casting/casting_utils.h" + +namespace paimon { +MapSharedShreddingFileReader::MapSharedShreddingFileReader( + std::unique_ptr&& reader, + std::map&& + shared_shredding_name_to_context, + const std::shared_ptr& pool) + : arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + shared_shredding_name_to_context_(std::move(shared_shredding_name_to_context)) {} + +Result> MapSharedShreddingFileReader::GetFileSchema() const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> physical_schema, + reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr physical_arrow_schema, + arrow::ImportSchema(physical_schema.get())); + + arrow::FieldVector logical_fields = physical_arrow_schema->fields(); + for (int32_t i = 0; i < physical_arrow_schema->num_fields(); ++i) { + const auto& field = physical_arrow_schema->field(i); + std::shared_ptr metadata = + std::const_pointer_cast(field->metadata()); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(logical_fields[i], ToLogicalMapField(field)); + } + + auto logical_schema = arrow::schema(std::move(logical_fields)); + auto c_logical_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*logical_schema, c_logical_schema.get())); + return c_logical_schema; +} + +Result> MapSharedShreddingFileReader::ToLogicalMapField( + const std::shared_ptr& physical_field) { + auto physical_type = std::dynamic_pointer_cast(physical_field->type()); + if (!physical_type) { + return Status::Invalid(fmt::format("shared-shredding field {} is not a physical struct", + physical_field->name())); + } + std::shared_ptr value_type; + bool value_nullable = true; + for (const auto& child : physical_type->fields()) { + if (child->name() == MapSharedShreddingDefine::kFieldMapping || + child->name() == MapSharedShreddingDefine::kOverflow) { + continue; + } + value_type = child->type(); + value_nullable = child->nullable(); + break; + } + if (!value_type) { + return Status::Invalid(fmt::format("cannot infer shared-shredding value type for field {}", + physical_field->name())); + } + return arrow::field( + physical_field->name(), + arrow::map(arrow::utf8(), arrow::field("value", value_type, value_nullable)), + physical_field->nullable()); +} + +Status MapSharedShreddingFileReader::SetReadSchema( + ::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (!read_schema) { + return Status::Invalid( + "invalid read schema in MapSharedShreddingFileReader, cannot be null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_read_schema, + arrow::ImportSchema(read_schema)); + std::vector shared_shredding_names; + for (const auto& field : logical_read_schema->fields()) { + if (shared_shredding_name_to_context_.find(field->name()) != + shared_shredding_name_to_context_.end()) { + shared_shredding_names.push_back(field->name()); + } + } + if (shared_shredding_names.empty()) { + // suppose not fall into MapSharedShreddingFileReader + return Status::Invalid("do not exist shared shredding columns in read schema"); + } + arrow::FieldVector resolved_fields = logical_read_schema->fields(); + for (const auto& name : shared_shredding_names) { + const auto& field = logical_read_schema->GetFieldByName(name); + if (!field) { + return Status::Invalid( + fmt::format("cannot find shared-shredding field {} in read schema", name)); + } + auto context_iter = shared_shredding_name_to_context_.find(field->name()); + if (context_iter == shared_shredding_name_to_context_.end()) { + return Status::Invalid( + fmt::format("cannot find shared-shredding metadata for field {}", field->name())); + } + std::set selected_physical_column_ids; + bool include_overflow = false; + for (const auto& selected_key : context_iter->second.selected_keys) { + // check if selected_key in file + auto name_iter = context_iter->second.meta.name_to_id.find(selected_key); + if (name_iter == context_iter->second.meta.name_to_id.end()) { + continue; + } + // check if selected_key in overflow_field + PAIMON_ASSIGN_OR_RAISE( + bool is_overflow_field, + MapSharedShreddingUtils::IsOverflowField(context_iter->second.meta, selected_key)); + include_overflow = include_overflow || is_overflow_field; + // check if selected_key in field_to_columns + auto column_iter = context_iter->second.meta.field_to_columns.find(name_iter->second); + if (column_iter == context_iter->second.meta.field_to_columns.end()) { + continue; + } + const std::vector& physical_column_ids = column_iter->second; + selected_physical_column_ids.insert(physical_column_ids.begin(), + physical_column_ids.end()); + } + std::shared_ptr resolved_type = + MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + context_iter->second.map_type->item_type(), selected_physical_column_ids, + context_iter->second.map_type->item_field()->nullable(), include_overflow); + resolved_fields[logical_read_schema->GetFieldIndex(name)] = + arrow::field(field->name(), resolved_type, field->nullable()); + } + auto resolved_schema = arrow::schema(std::move(resolved_fields)); + std::unique_ptr c_resolved_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*resolved_schema, c_resolved_schema.get())); + return reader_->SetReadSchema(c_resolved_schema.get(), predicate, selection_bitmap); +} + +Result MapSharedShreddingFileReader::NextBatch() { + return Status::Invalid( + "paimon inner reader MapSharedShreddingFileReader should use NextBatchWithBitmap"); +} + +Result MapSharedShreddingFileReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + reader_->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return batch_with_bitmap; + } + + auto& [batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (!struct_array) { + return Status::Invalid("cannot cast batch to StructArray in MapSharedShreddingFileReader"); + } + + arrow::ArrayVector resolved_arrays = struct_array->fields(); + arrow::FieldVector resolved_fields = struct_array->struct_type()->fields(); + for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { + const auto& physical_field = struct_array->struct_type()->field(field_idx); + auto iter = shared_shredding_name_to_context_.find(physical_field->name()); + if (iter == shared_shredding_name_to_context_.end()) { + continue; + } + auto physical_struct_array = + std::dynamic_pointer_cast(struct_array->field(field_idx)); + if (!physical_struct_array) { + return Status::Invalid(fmt::format( + "cannot cast physical shredding field {} to StructArray", physical_field->name())); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr logical_map_array, + RebuildLogicalMapArray(physical_field, physical_struct_array)); + resolved_arrays[field_idx] = logical_map_array; + resolved_fields[field_idx] = arrow::field(physical_field->name(), logical_map_array->type(), + physical_field->nullable()); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_struct_array, + arrow::StructArray::Make(resolved_arrays, resolved_fields)); + auto new_c_array = std::make_unique(); + auto new_c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*new_struct_array, new_c_array.get(), new_c_schema.get())); + batch = std::make_pair(std::move(new_c_array), std::move(new_c_schema)); + return batch_with_bitmap; +} + +Result> MapSharedShreddingFileReader::RebuildLogicalMapArray( + const std::shared_ptr& physical_field, + const std::shared_ptr& physical_struct_array) const { + std::string shredding_field_name = physical_field->name(); + auto iter = shared_shredding_name_to_context_.find(shredding_field_name); + if (iter == shared_shredding_name_to_context_.end()) { + return Status::Invalid( + fmt::format("cannot find shared-shredding context for field {}", shredding_field_name)); + } + const MapSharedShreddingFieldMeta& meta = iter->second.meta; + const std::vector& selected_keys = iter->second.selected_keys; + const auto& map_type = iter->second.map_type; + + auto field_mapping_array = std::dynamic_pointer_cast( + physical_struct_array->GetFieldByName(MapSharedShreddingDefine::kFieldMapping)); + if (!field_mapping_array) { + return Status::Invalid( + fmt::format("cannot find __field_mapping for field {}", shredding_field_name)); + } + auto field_mapping_values = + std::dynamic_pointer_cast(field_mapping_array->values()); + if (!field_mapping_values) { + return Status::Invalid("__field_mapping values is not an Int32Array"); + } + + auto selected_key_ids = ResolveSelectedKeyIds(meta, selected_keys); + std::map> physical_column_name_to_array; + std::shared_ptr overflow_array; + CollectPhysicalColumns(physical_struct_array, &physical_column_name_to_array, &overflow_array); + for (auto& [_, physical_column_array] : physical_column_name_to_array) { + if (physical_column_array->type_id() == arrow::Type::DICTIONARY) { + PAIMON_ASSIGN_OR_RAISE( + physical_column_array, + CastingUtils::Cast(physical_column_array, map_type->item_type(), + arrow::compute::CastOptions::Safe(), arrow_pool_.get())); + } + } + + std::shared_ptr overflow_keys; + std::shared_ptr overflow_items; + if (overflow_array) { + overflow_keys = + arrow::internal::checked_pointer_cast(overflow_array->keys()); + overflow_items = overflow_array->items(); + if (!overflow_keys || !overflow_items) { + return Status::Invalid("__overflow map has invalid key or item array"); + } + if (overflow_items->type_id() == arrow::Type::DICTIONARY) { + PAIMON_ASSIGN_OR_RAISE( + overflow_items, + CastingUtils::Cast(overflow_items, map_type->item_type(), + arrow::compute::CastOptions::Safe(), arrow_pool_.get())); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr map_builder_base, + arrow::MakeBuilder(map_type, arrow_pool_.get())); + auto* map_builder = dynamic_cast(map_builder_base.get()); + if (!map_builder) { + return Status::Invalid( + fmt::format("cannot create MapBuilder for field {}", shredding_field_name)); + } + auto* key_builder = dynamic_cast(map_builder->key_builder()); + if (!key_builder) { + return Status::Invalid(fmt::format("map key builder is not a StringBuilder for field {}", + shredding_field_name)); + } + arrow::ArrayBuilder* item_builder = map_builder->item_builder(); + if (!item_builder) { + return Status::Invalid( + fmt::format("map item builder is null for field {}", shredding_field_name)); + } + + int64_t row_count = physical_struct_array->length(); + int64_t max_item_count = row_count * static_cast(selected_key_ids.size()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Reserve(row_count)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Reserve(max_item_count)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(item_builder->Reserve(max_item_count)); + + for (int64_t row = 0; row < row_count; ++row) { + if (physical_struct_array->IsNull(row)) { + // null struct -> null map + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->AppendNull()); + continue; + } + if (field_mapping_array->IsNull(row)) { + return Status::Invalid(fmt::format( + "__field_mapping cannot be null in non-null shared-shredding row for field {}", + shredding_field_name)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Append()); + int32_t mapping_offset = field_mapping_array->value_offset(row); + int32_t mapping_length = field_mapping_array->value_length(row); + // follow the sequence in paimon.map.selected-keys + for (const auto& [selected_key, selected_field_id] : selected_key_ids) { + bool found = false; + for (int32_t pos = 0; pos < mapping_length; ++pos) { + int32_t mapping_index = mapping_offset + pos; + if (field_mapping_values->IsNull(mapping_index)) { + return Status::Invalid("__field_mapping element cannot be null"); + } + if (field_mapping_values->Value(mapping_index) != selected_field_id) { + continue; + } + std::string physical_column_name = + MapSharedShreddingDefine::PhysicalColumnName(pos); + auto physical_column_iter = + physical_column_name_to_array.find(physical_column_name); + if (physical_column_iter == physical_column_name_to_array.end()) { + return Status::Invalid( + fmt::format("cannot find selected physical column {} for field {}", + physical_column_name, shredding_field_name)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(selected_key)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + item_builder->AppendArraySlice(*physical_column_iter->second->data(), row, 1)); + found = true; + break; + } + if (found || !overflow_array) { + continue; + } + int32_t overflow_offset = overflow_array->value_offset(row); + int32_t overflow_length = overflow_array->value_length(row); + for (int32_t pos = 0; pos < overflow_length; ++pos) { + int32_t overflow_index = overflow_offset + pos; + if (!overflow_keys->IsNull(overflow_index) && + overflow_keys->Value(overflow_index) == selected_field_id) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(selected_key)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + item_builder->AppendArraySlice(*overflow_items->data(), overflow_index, 1)); + break; + } + } + } + } + std::shared_ptr map_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Finish(&map_array)); + return map_array; +} + +std::vector> MapSharedShreddingFileReader::ResolveSelectedKeyIds( + const MapSharedShreddingFieldMeta& meta, const std::vector& selected_keys) { + std::vector> selected_key_ids; + selected_key_ids.reserve(selected_keys.size()); + for (const auto& selected_key : selected_keys) { + auto id_iter = meta.name_to_id.find(selected_key); + if (id_iter == meta.name_to_id.end()) { + continue; + } + selected_key_ids.emplace_back(selected_key, id_iter->second); + } + return selected_key_ids; +} + +void MapSharedShreddingFileReader::CollectPhysicalColumns( + const std::shared_ptr& physical_struct_array, + std::map>* physical_column_name_to_array, + std::shared_ptr* overflow_array) { + const auto& struct_type = physical_struct_array->struct_type(); + for (int32_t i = 0; i < struct_type->num_fields(); ++i) { + const auto& sub_field = struct_type->field(i); + if (sub_field->name() == MapSharedShreddingDefine::kFieldMapping) { + continue; + } + if (sub_field->name() == MapSharedShreddingDefine::kOverflow) { + *overflow_array = arrow::internal::checked_pointer_cast( + physical_struct_array->field(i)); + continue; + } + (*physical_column_name_to_array)[sub_field->name()] = physical_struct_array->field(i); + } +} + +std::shared_ptr MapSharedShreddingFileReader::GetReaderMetrics() const { + return reader_->GetReaderMetrics(); +} + +void MapSharedShreddingFileReader::Close() { + reader_->Close(); +} + +Result MapSharedShreddingFileReader::GetPreviousBatchFileRowId( + uint64_t batch_row_id) const { + return reader_->GetPreviousBatchFileRowId(batch_row_id); +} + +Result MapSharedShreddingFileReader::GetNumberOfRows() const { + return reader_->GetNumberOfRows(); +} + +bool MapSharedShreddingFileReader::SupportPreciseBitmapSelection() const { + return reader_->SupportPreciseBitmapSelection(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h new file mode 100644 index 00000000..ba608cf0 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" + +namespace paimon { + +class MapSharedShreddingFileReader : public FileBatchReader { + public: + struct SharedShreddingContext { + SharedShreddingContext(const MapSharedShreddingFieldMeta& _meta, + const std::vector& _selected_keys, + const std::shared_ptr& _map_type) + : meta(_meta), selected_keys(_selected_keys), map_type(_map_type) {} + MapSharedShreddingFieldMeta meta; + std::vector selected_keys; + std::shared_ptr map_type; + }; + + MapSharedShreddingFileReader( + std::unique_ptr&& reader, + std::map&& shared_shredding_name_to_context, + const std::shared_ptr& pool); + + Result> GetFileSchema() const override; + + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; + + Result GetNumberOfRows() const override; + + bool SupportPreciseBitmapSelection() const override; + + private: + Result> RebuildLogicalMapArray( + const std::shared_ptr& physical_field, + const std::shared_ptr& physical_struct_array) const; + + static std::vector> ResolveSelectedKeyIds( + const MapSharedShreddingFieldMeta& meta, const std::vector& selected_keys); + + static void CollectPhysicalColumns( + const std::shared_ptr& physical_struct_array, + std::map>* physical_column_name_to_array, + std::shared_ptr* overflow_array); + + static Result> ToLogicalMapField( + const std::shared_ptr& physical_field); + + private: + std::shared_ptr arrow_pool_; + std::unique_ptr reader_; + std::map shared_shredding_name_to_context_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp new file mode 100644 index 00000000..c8ab3d83 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp @@ -0,0 +1,623 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/fs/external_path_provider.h" +#include "paimon/core/append/append_only_writer.h" +#include "paimon/core/compact/noop_compact_manager.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/commit_increment.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/record_batch.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/mock/mock_file_format_factory.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class MapSharedShreddingFileReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + static MapSharedShreddingFieldMeta TagsMeta() { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}, {3, {0}}, {4, {1}}}; + meta.overflow_field_set = {0, 2}; + meta.num_columns = 2; + meta.max_row_width = 4; + return meta; + } + + Result> PhysicalSchemaWithMetadata() const { + return PhysicalSchemaWithMetadata(TagsMeta()); + } + + Result> PhysicalSchemaWithMetadata( + const MapSharedShreddingFieldMeta& meta) const { + std::map field_to_num_columns = {{"tags", 2}}; + PAIMON_ASSIGN_OR_RAISE(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema_, field_to_num_columns)); + auto metadata = std::make_shared(); + PAIMON_RETURN_NOT_OK(MapSharedShreddingUtils::SerializeMetadata( + meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); + + arrow::FieldVector fields = physical_schema->fields(); + fields[1] = fields[1]->WithMetadata(metadata); + return arrow::schema(std::move(fields)); + } + + Result> PhysicalArray() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + PhysicalSchemaWithMetadata()); + std::string json = R"([ + [1, [[0, 1], 10, 20, null]], + [2, [[2, 0], 30, 40, null]], + [3, null], + [4, [[3, 4], 60, 70, [[0, 80], [2, null]]]] + ])"; + return arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), + json) + .ValueOrDie(); + } + + std::unique_ptr WrapReader( + std::unique_ptr&& reader, + const std::optional& selected_keys_str = std::nullopt) const { + EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + std::map + shared_shredding_name_to_context; + for (const auto& field : file_schema->fields()) { + auto metadata = std::const_pointer_cast(field->metadata()); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + continue; + } + EXPECT_OK_AND_ASSIGN(auto meta, MapSharedShreddingUtils::DeserializeMetadata(metadata)); + auto physical_type = + arrow::internal::checked_pointer_cast(field->type()); + std::shared_ptr item_field; + for (const auto& child : physical_type->fields()) { + if (child->name() != MapSharedShreddingDefine::kFieldMapping && + child->name() != MapSharedShreddingDefine::kOverflow) { + item_field = child; + break; + } + } + EXPECT_TRUE(item_field); + auto map_type = arrow::internal::checked_pointer_cast(arrow::map( + arrow::utf8(), arrow::field("value", item_field->type(), item_field->nullable()))); + std::vector selected_keys; + if (selected_keys_str.has_value()) { + selected_keys = StringUtils::Split(selected_keys_str.value(), ",", + /*ignore_empty=*/false); + } else { + selected_keys.reserve(meta.name_to_id.size()); + for (const auto& [key_name, _] : meta.name_to_id) { + selected_keys.push_back(key_name); + } + } + shared_shredding_name_to_context.emplace( + field->name(), MapSharedShreddingFileReader::SharedShreddingContext( + meta, selected_keys, map_type)); + } + return std::make_unique( + std::move(reader), std::move(shared_shredding_name_to_context), pool_); + } + + Result> CreateReader( + std::shared_ptr physical_array = nullptr, + std::shared_ptr physical_schema = nullptr, + const std::optional& selected_keys = std::nullopt) const { + if (!physical_schema) { + PAIMON_ASSIGN_OR_RAISE(physical_schema, PhysicalSchemaWithMetadata()); + } + if (!physical_array) { + PAIMON_ASSIGN_OR_RAISE(physical_array, PhysicalArray()); + } + auto mock_reader = std::make_unique( + physical_array, arrow::struct_(physical_schema->fields()), /*read_batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + return WrapReader(std::move(mock_reader), selected_keys); + } + + std::shared_ptr ReadSchema( + const std::optional& selected_keys) const { + arrow::FieldVector fields = logical_schema_->fields(); + if (selected_keys) { + auto metadata = std::make_shared(); + metadata->Append("paimon.map.selected-keys", *selected_keys); + fields[1] = fields[1]->WithMetadata(metadata); + } + return arrow::schema(std::move(fields)); + } + + std::unique_ptr ExportSchema(const std::shared_ptr& schema) const { + auto c_schema = std::make_unique(); + EXPECT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + return c_schema; + } + + std::unique_ptr CreateBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json) + .ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + EXPECT_OK_AND_ASSIGN(auto batch, batch_builder.Finish()); + return batch; + } + + void AssertChunkedArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) const { + ASSERT_TRUE(expected->Equals(actual)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); + } + + std::shared_ptr CreatePathFactory(const std::string& dir, + const std::string& format, + const CoreOptions& options) const { + auto path_factory = std::make_shared(); + EXPECT_OK(path_factory->Init(dir, format, options.DataFilePrefix(), nullptr)); + return path_factory; + } + + std::unique_ptr OpenFormatReader( + const std::string& file_path, const std::string& format, + const std::map& options = {}) const { + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format, options)); + EXPECT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + return reader_builder->Build(input_stream).value(); + } + + Result> CreateAppendOnlyWriter( + const CoreOptions& core_options, int64_t schema_id, + const std::shared_ptr& logical_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager) const { + return std::make_unique(core_options, schema_id, logical_schema, + write_cols, max_sequence_number, path_factory, + compact_manager, pool_); + } + + private: + std::shared_ptr pool_; + std::shared_ptr logical_schema_ = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map options_ = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }; +}; + +TEST_F(MapSharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) { + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader()); + + ASSERT_OK_AND_ASSIGN(auto c_schema, reader->GetFileSchema()); + auto schema = arrow::ImportSchema(c_schema.get()).ValueOrDie(); + + ASSERT_TRUE(schema->Equals(logical_schema_, /*check_metadata=*/false)) + << "Expected:\n" + << logical_schema_->ToString() << "\nActual:\n" + << schema->ToString(); + ASSERT_FALSE(schema->field(1)->HasMetadata()); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"b")); + auto read_schema = ExportSchema(ReadSchema("b")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["b", 20]]], + [2, []], + [3, null], + [4, []] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c")); + auto read_schema = ExportSchema(ReadSchema("a,c")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c,missing")); + auto read_schema = ExportSchema(ReadSchema("a,c,missing")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader()); + auto read_schema = ExportSchema(ReadSchema(std::nullopt)); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null], ["d", 60], ["e", 70]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"", 0}, {" ", 1}, {".", 2}, {"a", 3}}; + meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}, {3, {1}}}; + meta.num_columns = 2; + meta.max_row_width = 2; + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata(meta)); + std::string json = R"([ + [1, [[0, 1], 10, 20, null]], + [2, [[2, 3], 30, 40, null]], + [3, null] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + + auto assert_read = [&](const std::string& selected_keys, const std::string& expected_json) { + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, selected_keys)); + auto read_schema = ExportSchema(ReadSchema(selected_keys)); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {expected_json}, &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); + }; + + assert_read(" ", R"([ + [1, [[" ", 20]]], + [2, []], + [3, null] + ])"); + assert_read(".", R"([ + [1, []], + [2, [[".", 30]]], + [3, null] + ])"); + assert_read("a,", R"([ + [1, [["", 10]]], + [2, [["a", 40]]], + [3, null] + ])"); + assert_read("", R"([ + [1, [["", 10]]], + [2, []], + [3, null] + ])"); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"missing")); + auto read_schema = ExportSchema(ReadSchema("missing")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, []], + [2, []], + [3, null], + [4, []] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata()); + std::string json = R"([ + [1, [null, 10, null, null]] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, /*selected_keys=*/"a")); + auto read_schema = ExportSchema(ReadSchema("a")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(reader.get()), + "__field_mapping cannot be null"); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata()); + std::string json = R"([ + [1, [[0, null], 10, null, null]] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, /*selected_keys=*/"b")); + auto read_schema = ExportSchema(ReadSchema("b")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(reader.get()), + "__field_mapping element cannot be null"); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestListValue) { + std::shared_ptr logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), + }); + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {0}}, {2, {0}}}; + meta.overflow_field_set = {2}; + meta.num_columns = 2; + meta.max_row_width = 3; + + std::map field_to_num_columns = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata( + meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); + arrow::FieldVector physical_fields = physical_schema->fields(); + physical_fields[1] = physical_fields[1]->WithMetadata(metadata); + physical_schema = arrow::schema(std::move(physical_fields)); + + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), R"([ + [1, [[0, 1], [1, null, 2], [3], null]], + [2, [[2, 0], [5, 6], [7], null]], + [3, null], + [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, + /*selected_keys=*/"a,c")); // NOLINT(whitespace/comma) + + auto read_metadata = std::make_shared(); + read_metadata->Append("paimon.map.selected-keys", "a,c"); + arrow::FieldVector read_fields = logical_schema->fields(); + read_fields[1] = read_fields[1]->WithMetadata(read_metadata); + auto read_schema = ExportSchema(arrow::schema(std::move(read_fields))); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema->fields()), {R"([ + [1, [["a", [1, null, 2]]]], + [2, [["a", [7]], ["c", [5, 6]]]], + [3, null], + [4, [["a", [9, 10]], ["c", [null]]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { + std::shared_ptr logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }); + auto options = options_; + std::string format = "orc"; + options[Options::FILE_FORMAT] = format; + options["orc.dictionary-key-size-threshold"] = "1"; + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, logical_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + auto path_factory = CreatePathFactory(dir->Str(), format, core_options); + auto compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto writer, + CreateAppendOnlyWriter(core_options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager)); + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", "red"], ["b", "blue"]]], + [2, [["c", "green"], ["a", "red"], ["b", "blue"]]], + [3, null], + [4, [["d", "yellow"], ["e", "blue"], ["c", null], ["a", "red"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + std::map reader_options = {{"orc.read.enable-lazy-decoding", "true"}}; + auto reader = WrapReader(OpenFormatReader(data_file_path, format, reader_options), + /*selected_keys_str=*/"a,c"); + + auto read_metadata = std::make_shared(); + read_metadata->Append("paimon.map.selected-keys", "a,c"); + arrow::FieldVector read_fields = logical_schema->fields(); + read_fields[1] = read_fields[1]->WithMetadata(read_metadata); + auto read_schema = ExportSchema(arrow::schema(std::move(read_fields))); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema->fields()), {R"([ + [1, [["a", "red"]]], + [2, [["a", "red"], ["c", "green"]]], + [3, null], + [4, [["a", "red"], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(MapSharedShreddingFileReaderTest, TestReadsRealFormatFile) { + // TODO(lisizhuo.lsz): support other format + auto options = options_; + std::string format = "orc"; + options[Options::FILE_FORMAT] = format; + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, logical_schema_, + /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + auto path_factory = CreatePathFactory(dir->Str(), format, core_options); + auto compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto writer, + CreateAppendOnlyWriter(core_options, /*schema_id=*/0, logical_schema_, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager)); + auto batch = CreateBatch(logical_schema_, R"([ + [1, [["a", 1], ["b", 2]]], + [2, [["c", 3], ["a", 4], ["b", 5]]], + [3, null], + [4, [["d", 6], ["e", 7], ["c", null], ["a", 9]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + auto reader = WrapReader(OpenFormatReader(data_file_path, format), + /*selected_keys_str=*/"a,c"); + + auto read_schema = ExportSchema(ReadSchema("a,c")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 1]]], + [2, [["a", 4], ["c", 3]]], + [3, null], + [4, [["a", 9], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp new file mode 100644 index 00000000..c8dc5412 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" + +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema> logical_schema, + const std::map& field_to_num_columns) { + if (!logical_schema) { + return Status::Invalid("logical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_logical_schema, + arrow::ImportSchema(logical_schema.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(arrow_logical_schema, + field_to_num_columns)); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*physical_schema, c_schema.get())); + return c_schema; +} + +Result> MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema> physical_schema, + const std::map& field_name_to_meta, + const std::string& compression) { + if (!physical_schema) { + return Status::Invalid("physical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_physical_schema, + arrow::ImportSchema(physical_schema.get())); + + arrow::FieldVector updated_fields = arrow_physical_schema->fields(); + for (const auto& [field_name, field_meta] : field_name_to_meta) { + int32_t field_index = arrow_physical_schema->GetFieldIndex(field_name); + if (field_index < 0) { + return Status::Invalid(fmt::format( + "Shared-shredding field '{}' not found in physical schema.", field_name)); + } + + const auto& field = arrow_physical_schema->field(field_index); + auto metadata = field->metadata() ? field->metadata()->Copy() + : std::make_shared(); + PAIMON_RETURN_NOT_OK( + MapSharedShreddingUtils::SerializeMetadata(field_meta, compression, metadata.get())); + updated_fields[field_index] = field->WithMetadata(metadata); + } + + auto updated_schema = + arrow::schema(std::move(updated_fields), arrow_physical_schema->metadata()); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*updated_schema, c_schema.get())); + return c_schema; +} + +Result MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name) { + if (!physical_schema) { + return Status::Invalid("physical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_physical_schema, + arrow::ImportSchema(physical_schema.get())); + const auto& field = arrow_physical_schema->GetFieldByName(field_name); + if (!field) { + return Status::Invalid( + fmt::format("Shared-shredding field '{}' not found in physical schema.", field_name)); + } + + auto metadata = + field->metadata() ? field->metadata()->Copy() : std::shared_ptr(); + return MapSharedShreddingUtils::DeserializeMetadata(metadata); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp new file mode 100644 index 00000000..e421bc3f --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaBasic) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto id_metadata = std::make_shared(); + id_metadata->Append("paimon.field.id", "1"); + auto schema_metadata = std::make_shared(); + schema_metadata->Append("schema.key", "schema.value"); + auto schema = arrow::schema( + {arrow::field("id", arrow::int32(), true, id_metadata), + arrow::field("tags", arrow::struct_({arrow::field("__field_mapping", + arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}))}, + schema_metadata); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + ASSERT_TRUE(c_updated_schema); + ASSERT_TRUE(c_updated_schema->release); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + + ASSERT_TRUE(updated_schema->metadata()->Equals(*schema_metadata)); + ASSERT_TRUE(updated_schema->field(0)->metadata()->Equals(*id_metadata)); + + auto tags_metadata = updated_schema->GetFieldByName("tags")->metadata()->Copy(); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(tags_metadata)); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(tags_metadata)); + ASSERT_EQ(deserialized, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaInvalidInput) { + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema>(), {}, "none"), + "physical schema is null"); + + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}}; + tags_meta.field_to_columns = {{0, {0}}}; + tags_meta.num_columns = 1; + tags_meta.max_row_width = 1; + + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none"), + "Shared-shredding field 'tags' not found in physical schema."); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaPreservesExistingFieldMetadata) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}}; + tags_meta.field_to_columns = {{0, {0}}}; + tags_meta.num_columns = 1; + tags_meta.max_row_width = 1; + + auto tags_metadata = std::make_shared(); + tags_metadata->Append("paimon.field.id", "7"); + tags_metadata->Append("description", "original tags field"); + auto schema = arrow::schema({arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true)}), + true, tags_metadata)}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + ASSERT_TRUE(c_updated_schema); + ASSERT_TRUE(c_updated_schema->release); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + + auto updated_metadata = updated_schema->field(0)->metadata()->Copy(); + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("paimon.field.id")), "7"); + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("description")), + "original tags field"); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(updated_metadata)); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaOverwritesExistingShreddingMetadata) { + MapSharedShreddingFieldMeta old_meta; + old_meta.name_to_id = {{"old", 0}}; + old_meta.field_to_columns = {{0, {0}}}; + old_meta.num_columns = 1; + old_meta.max_row_width = 1; + + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.overflow_field_set = {1}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto tags_metadata = std::make_shared(); + tags_metadata->Append("paimon.field.id", "7"); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(old_meta, "none", tags_metadata.get())); + auto schema = arrow::schema({arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}), + true, tags_metadata)}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + auto updated_metadata = updated_schema->field(0)->metadata()->Copy(); + + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("paimon.field.id")), "7"); + int32_t storage_layout_key_count = 0; + for (const auto& key : updated_metadata->keys()) { + if (key == MapShreddingDefine::kStorageLayout) { + ++storage_layout_key_count; + } + } + ASSERT_EQ(storage_layout_key_count, 1); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(updated_metadata)); + ASSERT_EQ(deserialized, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromField) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(tags_meta, "none", metadata.get())); + auto field = arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}), + true, metadata); + auto schema = arrow::schema({arrow::field("id", arrow::int32()), field}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto parsed_meta, MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::move(c_schema), "tags")); + ASSERT_EQ(parsed_meta, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldNoShreddingMetadata) { + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("tags", arrow::utf8())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG( + MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), "tags"), + "metadata is null or storage layout is not shared-shredding"); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldInvalidInput) { + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema>(), "tags"), + "physical schema is null"); + + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG( + MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), "tags"), + "Shared-shredding field 'tags' not found in physical schema."); +} + +TEST(MapSharedShreddingSchemaUtilsTest, LogicalToPhysicalSchemaInvalidInput) { + std::map field_to_num_columns = {{"tags", 2}}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema>(), field_to_num_columns), + "logical schema is null"); + + auto schema = + arrow::schema({arrow::field("ts", arrow::int64()), arrow::field("tags", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::move(c_schema), field_to_num_columns), + "Field 'tags' is expected to be MAP type"); +} + +TEST(MapSharedShreddingSchemaUtilsTest, LogicalToPhysicalSchemaNestedListValue) { + // MAP, b: array, c: array>> + auto nested_value = arrow::struct_({arrow::field("a", arrow::list(arrow::int32())), + arrow::field("b", arrow::list(arrow::int32())), + arrow::field("c", arrow::list(arrow::int32()))}); + auto map_type = arrow::map(arrow::utf8(), nested_value); + auto schema = + arrow::schema({arrow::field("ts", arrow::int64()), arrow::field("data", map_type)}); + + std::map field_to_num_columns = {{"data", 3}}; + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_physical_schema, + MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::move(c_schema), field_to_num_columns)); + ASSERT_TRUE(c_physical_schema); + ASSERT_TRUE(c_physical_schema->release); + auto physical_schema = arrow::ImportSchema(c_physical_schema.get()).ValueOrDie(); + + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", nested_value, true), + arrow::field("__col_1", nested_value, true), + arrow::field("__col_2", nested_value, true), + arrow::field("__overflow", arrow::map(arrow::int32(), nested_value), true), + }); + auto expected_schema = arrow::schema( + {arrow::field("ts", arrow::int64()), arrow::field("data", expected_struct, true)}); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp new file mode 100644 index 00000000..6ba2b43c --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" + +#include +#include + +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/compression/block_compression_factory.h" +#include "paimon/common/compression/block_compressor.h" +#include "paimon/common/compression/block_decompressor.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/map_storage_layout.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { +Result> MapSharedShreddingUtils::GetPhysicalColumnIndices( + const MapSharedShreddingFieldMeta& meta, const std::string& name) { + auto name_iter = meta.name_to_id.find(name); + if (name_iter == meta.name_to_id.end()) { + return Status::Invalid( + fmt::format("cannot find field {} in map shared shredding meta", name)); + } + auto id_iter = meta.field_to_columns.find(name_iter->second); + if (id_iter == meta.field_to_columns.end()) { + return Status::Invalid( + fmt::format("cannot find field id {} in field_to_columns in map shared shredding meta", + name_iter->second)); + } + return id_iter->second; +} + +// ---- Column detection ---- +bool MapSharedShreddingUtils::IsShreddingKeyMap( + const std::shared_ptr& arrow_type) { + if (arrow_type->id() != arrow::Type::MAP) { + return false; + } + auto map_type = std::static_pointer_cast(arrow_type); + return map_type->key_type()->id() == arrow::Type::STRING; +} + +Result> MapSharedShreddingUtils::DetectShreddingColumns( + const std::shared_ptr& schema, const CoreOptions& options) { + std::vector field_names; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + const auto& field = schema->field(i); + if (!IsShreddingKeyMap(field->type())) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(MapStorageLayout layout, options.GetMapStorageLayout(field->name())); + if (layout == MapStorageLayout::SHARED_SHREDDING) { + field_names.push_back(field->name()); + } + } + return field_names; +} + +// ---- Schema conversion ---- +std::shared_ptr MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::set& physical_col_ids, + bool value_nullable, bool include_overflow) { + std::vector sorted_cols(physical_col_ids.begin(), physical_col_ids.end()); + return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, + include_overflow); +} + +std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructType( + const std::shared_ptr& value_type, int32_t num_columns, bool value_nullable) { + std::vector sorted_cols(num_columns); + std::iota(sorted_cols.begin(), sorted_cols.end(), 0); + return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, + /*include_overflow=*/true); +} + +std::shared_ptr MapSharedShreddingUtils::InnerBuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::vector& sorted_cols, + bool value_nullable, bool include_overflow) { + arrow::FieldVector struct_fields; + struct_fields.reserve(sorted_cols.size() + 2); + struct_fields.push_back( + arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), true)); + for (const auto& col : sorted_cols) { + struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(col), + value_type, value_nullable)); + } + if (include_overflow) { + struct_fields.push_back(arrow::field( + MapSharedShreddingDefine::kOverflow, + arrow::map(arrow::int32(), arrow::field("value", value_type, value_nullable)), true)); + } + return arrow::struct_(std::move(struct_fields)); +} + +Result> MapSharedShreddingUtils::LogicalToPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::map& field_to_num_columns) { + arrow::FieldVector physical_fields; + physical_fields.reserve(logical_schema->num_fields()); + + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + const auto& field = logical_schema->field(i); + auto it = field_to_num_columns.find(field->name()); + if (it != field_to_num_columns.end()) { + if (field->type()->id() != arrow::Type::MAP) { + return Status::Invalid( + fmt::format("Field '{}' is expected to be MAP type, but got '{}'.", + field->name(), field->type()->name())); + } + auto map_type = std::static_pointer_cast(field->type()); + auto value_type = map_type->item_type(); + bool value_nullable = map_type->item_field()->nullable(); + auto physical_type = BuildPhysicalStructType(value_type, it->second, value_nullable); + auto physical_field = field->WithType(physical_type); + physical_fields.push_back(physical_field); + } else { + physical_fields.push_back(field); + } + } + + return arrow::schema(std::move(physical_fields)); +} + +Result> MapSharedShreddingUtils::BuildColumnToNumColumns( + const std::vector& shredding_field_names, const CoreOptions& options) { + std::map field_to_num_columns; + for (const std::string& field_name : shredding_field_names) { + PAIMON_ASSIGN_OR_RAISE(int32_t max_columns, + options.GetMapSharedShreddingMaxColumns(field_name)); + field_to_num_columns[field_name] = max_columns; + } + return field_to_num_columns; +} + +// ---- Metadata serialization helpers ---- + +namespace { + +std::string JsonEncodeObject( + std::function builder) { + rapidjson::Document doc(rapidjson::kObjectType); + auto& allocator = doc.GetAllocator(); + builder(&doc, &allocator); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + return buffer.GetString(); +} + +std::string JsonEncodeArray( + std::function builder) { + rapidjson::Document doc(rapidjson::kArrayType); + auto& allocator = doc.GetAllocator(); + builder(&doc, &allocator); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + return buffer.GetString(); +} + +Result CompressString(const std::string& input, const std::string& compression) { + CompressOptions compress_opts{compression, /*zstd_level=*/1}; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr factory, + BlockCompressionFactory::Create(compress_opts)); + std::shared_ptr compressor = factory->GetCompressor(); + if (!compressor) { + return input; + } + + auto src_size = static_cast(input.size()); + int32_t max_compressed = compressor->GetMaxCompressedSize(src_size); + std::string output(max_compressed, '\0'); + + PAIMON_ASSIGN_OR_RAISE( + int32_t actual_size, + compressor->Compress(input.data(), src_size, output.data(), max_compressed)); + + output.resize(actual_size); + return output; +} + +Result DecompressString(const std::string& input, int32_t original_len, + const std::string& compression) { + CompressOptions compress_opts{compression, /*zstd_level=*/1}; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr factory, + BlockCompressionFactory::Create(compress_opts)); + std::shared_ptr decompressor = factory->GetDecompressor(); + if (!decompressor) { + return input; + } + std::string output(original_len, '\0'); + PAIMON_ASSIGN_OR_RAISE( + int32_t decompressed_len, + decompressor->Decompress(input.data(), static_cast(input.size()), output.data(), + original_len)); + output.resize(decompressed_len); + return output; +} + +Result GetRequiredValue(const std::shared_ptr& metadata, + const std::string& key) { + int32_t index = metadata->FindKey(key); + if (index < 0) { + return Status::Invalid(fmt::format("missing shredding metadata key: {}", key)); + } + return metadata->value(index); +} + +Result GetRequiredInt32(const std::shared_ptr& metadata, + const std::string& key) { + PAIMON_ASSIGN_OR_RAISE(std::string value, GetRequiredValue(metadata, key)); + std::optional parsed = StringUtils::StringToValue(value); + if (!parsed.has_value()) { + return Status::Invalid(fmt::format("malformed shredding metadata value for key: {}", key)); + } + return parsed.value(); +} + +std::string SerializeFieldDict(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeObject([&](rapidjson::Document* doc, + rapidjson::Document::AllocatorType* alloc) { + for (const auto& [name, id] : field_meta.name_to_id) { + doc->AddMember(rapidjson::Value(name.c_str(), *alloc), rapidjson::Value(id), *alloc); + } + }); +} + +std::string SerializeFieldColumns(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeObject( + [&](rapidjson::Document* doc, rapidjson::Document::AllocatorType* alloc) { + for (const auto& [field_id, col_vec] : field_meta.field_to_columns) { + rapidjson::Value array(rapidjson::kArrayType); + std::vector sorted_cols(col_vec.begin(), col_vec.end()); + std::sort(sorted_cols.begin(), sorted_cols.end()); + for (int32_t col : sorted_cols) { + array.PushBack(col, *alloc); + } + std::string key = std::to_string(field_id); + doc->AddMember(rapidjson::Value(key.c_str(), *alloc), array, *alloc); + } + }); +} + +std::string SerializeOverflowSet(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeArray( + [&](rapidjson::Document* doc, rapidjson::Document::AllocatorType* alloc) { + std::vector sorted(field_meta.overflow_field_set.begin(), + field_meta.overflow_field_set.end()); + std::sort(sorted.begin(), sorted.end()); + for (int32_t field_id : sorted) { + doc->PushBack(field_id, *alloc); + } + }); +} + +/// Safe JSON integer extraction with error propagation. +Result JsonGetInt(const rapidjson::Value& val, const char* context_msg) { + if (!val.IsInt()) { + return Status::Invalid(fmt::format("malformed shredding metadata: {}", context_msg)); + } + return val.GetInt(); +} + +Result> DeserializeFieldDict(const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsObject()) { + return Status::Invalid("malformed shredding field_dict metadata"); + } + std::map name_to_id; + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, JsonGetInt(it->value, "field_dict value is not int")); + name_to_id[it->name.GetString()] = id; + } + return name_to_id; +} + +Result>> DeserializeFieldColumns( + const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsObject()) { + return Status::Invalid("malformed shredding field_columns metadata"); + } + std::map> field_to_columns; + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + std::optional field_id = StringUtils::StringToValue(it->name.GetString()); + if (!field_id.has_value()) { + return Status::Invalid("malformed shredding field_columns: invalid field_id key"); + } + const auto& array = it->value; + if (!array.IsArray()) { + return Status::Invalid("malformed shredding field_columns: value is not array"); + } + std::vector cols; + cols.reserve(array.Size()); + for (rapidjson::SizeType i = 0; i < array.Size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t col, + JsonGetInt(array[i], "field_columns element is not int")); + cols.push_back(col); + } + field_to_columns[field_id.value()] = std::move(cols); + } + return field_to_columns; +} + +Result> DeserializeOverflowSet(const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsArray()) { + return Status::Invalid("malformed shredding overflow_set metadata"); + } + std::set overflow_set; + for (rapidjson::SizeType i = 0; i < doc.Size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, + JsonGetInt(doc[i], "overflow_set element is not int")); + overflow_set.insert(field_id); + } + return overflow_set; +} + +} // namespace + +Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata) { + const std::string normalized_compression = StringUtils::ToLowerCase(compression); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set( + MapShreddingDefine::kStorageLayout, MapShreddingDefine::kStorageLayoutSharedShredding)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kVersion, + std::to_string(MapSharedShreddingDefine::kCurrentVersion))); + + std::string field_dict_json = SerializeFieldDict(field_meta); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kFieldDictOriginalSize, + std::to_string(field_dict_json.size()))); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kFieldDictCompression, normalized_compression)); + PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict, + CompressString(field_dict_json, normalized_compression)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kFieldDict, std::move(compressed_dict))); + + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kFieldColumns, SerializeFieldColumns(field_meta))); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kOverflowSet, SerializeOverflowSet(field_meta))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kNumColumns, + std::to_string(field_meta.num_columns))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kMaxRowWidth, + std::to_string(field_meta.max_row_width))); + + return Status::OK(); +} + +Result MapSharedShreddingUtils::DeserializeMetadata( + const std::shared_ptr& metadata) { + if (!HasShreddingMetadata(metadata)) { + return Status::Invalid("metadata is null or storage layout is not shared-shredding"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t version, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kVersion)); + if (version != MapSharedShreddingDefine::kCurrentVersion) { + return Status::Invalid( + fmt::format("unsupported shared-shredding metadata version: {}, expected: {}", version, + MapSharedShreddingDefine::kCurrentVersion)); + } + + MapSharedShreddingFieldMeta result; + + // field_dict (compressed) + PAIMON_ASSIGN_OR_RAISE( + int32_t original_len, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kFieldDictOriginalSize)); + PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict, + GetRequiredValue(metadata, MapSharedShreddingDefine::kFieldDict)); + int32_t compression_index = metadata->FindKey(MapSharedShreddingDefine::kFieldDictCompression); + std::string field_dict_compression = compression_index < 0 + ? MapSharedShreddingDefine::kDefaultDictCompression + : metadata->value(compression_index); + field_dict_compression = StringUtils::ToLowerCase(field_dict_compression); + PAIMON_ASSIGN_OR_RAISE(std::string field_dict_json, + DecompressString(compressed_dict, original_len, field_dict_compression)); + PAIMON_ASSIGN_OR_RAISE(result.name_to_id, DeserializeFieldDict(field_dict_json)); + + // field_columns + PAIMON_ASSIGN_OR_RAISE(std::string field_columns_json, + GetRequiredValue(metadata, MapSharedShreddingDefine::kFieldColumns)); + PAIMON_ASSIGN_OR_RAISE(result.field_to_columns, DeserializeFieldColumns(field_columns_json)); + + // overflow_set + PAIMON_ASSIGN_OR_RAISE(std::string overflow_json, + GetRequiredValue(metadata, MapSharedShreddingDefine::kOverflowSet)); + PAIMON_ASSIGN_OR_RAISE(result.overflow_field_set, DeserializeOverflowSet(overflow_json)); + + // num_columns & max_row_width + PAIMON_ASSIGN_OR_RAISE(result.num_columns, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kNumColumns)); + PAIMON_ASSIGN_OR_RAISE(result.max_row_width, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kMaxRowWidth)); + + return result; +} + +bool MapSharedShreddingUtils::HasShreddingMetadata( + const std::shared_ptr& metadata) { + if (!metadata) { + return false; + } + auto index = metadata->FindKey(MapShreddingDefine::kStorageLayout); + if (index < 0) { + return false; + } + return metadata->value(index) == MapShreddingDefine::kStorageLayoutSharedShredding; +} + +Result MapSharedShreddingUtils::IsOverflowField(const MapSharedShreddingFieldMeta& meta, + const std::string& name) { + auto name_iter = meta.name_to_id.find(name); + if (name_iter == meta.name_to_id.end()) { + return Status::Invalid( + fmt::format("cannot find field {} in map shared shredding meta", name)); + } + return meta.overflow_field_set.count(name_iter->second) > 0; +} + +std::function>()> +MapSharedShreddingUtils::BuildMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression, const std::shared_ptr& physical_schema) { + return [converter, compression, physical_schema]() -> Result> { + const std::vector& shredding_field_names = + converter->GetShreddingColumnNames(); + arrow::FieldVector updated_fields = physical_schema->fields(); + for (const std::string& field_name : shredding_field_names) { + int32_t col_index = physical_schema->GetFieldIndex(field_name); + if (col_index < 0) { + return Status::Invalid(fmt::format( + "Shared-shredding field '{}' not found in physical schema.", field_name)); + } + const auto& field = physical_schema->field(col_index); + auto metadata = field->metadata() ? field->metadata()->Copy() + : std::make_shared(); + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta file_meta, + converter->BuildFieldMeta(field_name)); + PAIMON_RETURN_NOT_OK( + MapSharedShreddingUtils::SerializeMetadata(file_meta, compression, metadata.get())); + updated_fields[col_index] = field->WithMetadata(metadata); + } + return arrow::schema(std::move(updated_fields)); + }; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h new file mode 100644 index 00000000..00351b65 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class KeyValueMetadata; +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class MapSharedShreddingBatchConverter; + +/// Utility functions for shared-shredding MAP storage layout. +class MapSharedShreddingUtils { + public: + MapSharedShreddingUtils() = delete; + ~MapSharedShreddingUtils() = delete; + + // ---- Column detection ---- + + /// Checks whether a given arrow field is MAP (the type prerequisite for shredding). + /// @param arrow_type The Arrow data type of the column. + /// @return true if the type is MAP. + static bool IsShreddingKeyMap(const std::shared_ptr& arrow_type); + + /// Finds all shredding MAP field names in a schema by checking per-column config + /// via CoreOptions. + /// @param schema The logical Arrow schema. + /// @param options CoreOptions containing per-column configuration. + /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error + /// if validation fails. + static Result> DetectShreddingColumns( + const std::shared_ptr& schema, const CoreOptions& options); + + /// Builds shared-shredding max column counts from DetectShreddingColumns result and + /// CoreOptions. + /// @param shredding_field_names Field names returned by DetectShreddingColumns. + /// @param options CoreOptions containing per-column shared-shredding config. + /// @return Map from field name to its configured maximum physical width. + static Result> BuildColumnToNumColumns( + const std::vector& shredding_field_names, const CoreOptions& options); + + // ---- Schema conversion ---- + + /// Converts a logical schema to a physical schema by replacing shredding MAP columns + /// with their physical Struct representation. + /// @param logical_schema The original schema with MAP columns. + /// @param field_to_num_columns Map from field name to its physical column count K. + /// Each shredding column can have its own width. + /// @return The physical schema for file writing. + static Result> LogicalToPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::map& field_to_num_columns); + + /// Builds the physical Arrow type for one shredding MAP column with physical_col_ids. + /// @param value_type The value type of the original MAP. + /// @param physical_col_ids The set of physical column ids to include. + /// @param value_nullable Whether the MAP's value field is nullable. + /// @param include_overflow Whether to include __overflow column. + static std::shared_ptr BuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, + const std::set& physical_col_ids, bool value_nullable, bool include_overflow); + + // ---- Metadata serialization ---- + + /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. + /// @param field_meta The field-level shredding metadata to serialize. + /// @param compression Compression codec name for field_dict compression. + /// @param[out] metadata The KeyValueMetadata to append entries to. + static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata); + + /// Deserializes shredding metadata from file footer KeyValueMetadata (per field). + /// @param metadata The KeyValueMetadata from file footer. + /// @return Parsed MapSharedShreddingFieldMeta, or error if metadata is missing/malformed. + static Result DeserializeMetadata( + const std::shared_ptr& metadata); + + /// Checks whether a KeyValueMetadata contains shredding MAP metadata. + static bool HasShreddingMetadata(const std::shared_ptr& metadata); + + /// Checks whether a field in MapSharedShreddingFieldMeta is a overflow field. + static Result IsOverflowField(const MapSharedShreddingFieldMeta& meta, + const std::string& name); + + // ---- Writer helpers ---- + + /// Builds a MetadataFinalizer that serializes shredding metadata into per-field + /// KeyValueMetadata. + /// Shared by DataFileWriter (append-only) and KeyValueDataFileWriter (PK table). + /// @param converter The batch converter that holds field-dict state for BuildFieldMeta. + /// @param compression Compression codec name for field_dict serialization (e.g. "zstd"). + /// @param physical_schema The physical schema used for writing. + /// @return A callable that produces the updated schema with shredding metadata. + static std::function>()> BuildMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression, const std::shared_ptr& physical_schema); + + private: + /// Returns the physical column indices for the given field name from the shredding meta. + /// @param meta The shredding field meta parsed from file footer. + /// @param name The field name to look up. + /// @return Vector of physical column indices assigned to this field, + /// or Status::Invalid if the field name or field id is not found. + static Result> GetPhysicalColumnIndices( + const MapSharedShreddingFieldMeta& meta, const std::string& name); + + /// Builds the physical Arrow type for one shredding MAP column. + /// @param value_type The value type of the original MAP. + /// @param num_columns Number of physical columns K. + /// @param value_nullable Whether the MAP's value field is nullable. + static std::shared_ptr BuildPhysicalStructType( + const std::shared_ptr& value_type, int32_t num_columns, + bool value_nullable); + + /// Builds the physical Arrow type for one shredding MAP column with sorted_cols. + /// @param value_type The value type of the original MAP. + /// @param sorted_cols The vector of physical column ids to include. + /// @param value_nullable Whether the MAP's value field is nullable. + /// @param include_overflow Whether to include __overflow column. + static std::shared_ptr InnerBuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::vector& sorted_cols, + bool value_nullable, bool include_overflow); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp new file mode 100644 index 00000000..85467e48 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" + +#include + +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +// ---- IsShreddingKeyMap ---- + +TEST(MapSharedShreddingUtilsTest, IsShreddingKeyMap) { + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), arrow::int32()))); + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), arrow::float64()))); + // Nested value type (struct) + auto nested_value = + arrow::struct_({arrow::field("x", arrow::int32()), arrow::field("y", arrow::utf8())}); + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), nested_value))); + ASSERT_FALSE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::int32(), arrow::utf8()))); + ASSERT_FALSE(MapSharedShreddingUtils::IsShreddingKeyMap(arrow::int32())); + ASSERT_FALSE(MapSharedShreddingUtils::IsShreddingKeyMap(arrow::list(arrow::utf8()))); +} + +// ---- DetectShreddingColumns ---- + +TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsBasic) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("metrics", arrow::map(arrow::utf8(), arrow::float64())), + arrow::field("name", arrow::utf8()), + }); + + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}})); + + ASSERT_OK_AND_ASSIGN(auto field_names, + MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); + ASSERT_EQ(field_names.size(), 2); + ASSERT_EQ(field_names[0], "tags"); + ASSERT_EQ(field_names[1], "metrics"); +} + +TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsNoShredding) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }); + + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_OK_AND_ASSIGN(auto field_names, + MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); + ASSERT_TRUE(field_names.empty()); +} + +// ---- LogicalToPhysicalSchema ---- + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaBasic) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("name", arrow::utf8()), + }); + + std::map field_to_num_columns = {{"tags", 4}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, field_to_num_columns)); + + // Build expected schema for comparison + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true), + arrow::field("__col_2", arrow::utf8(), true), + arrow::field("__col_3", arrow::utf8(), true), + arrow::field("__overflow", arrow::map(arrow::int32(), arrow::utf8()), true), + }); + auto expected_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", expected_struct, true), + arrow::field("name", arrow::utf8()), + }); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNestedValue) { + // MAP> + auto nested_value = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + auto map_type = arrow::map(arrow::utf8(), nested_value); + auto schema = arrow::schema({arrow::field("data", map_type)}); + + std::map field_to_num_columns = {{"data", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, field_to_num_columns)); + + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", nested_value, true), + arrow::field("__col_1", nested_value, true), + arrow::field("__overflow", arrow::map(arrow::int32(), nested_value), true), + }); + auto expected_schema = arrow::schema({arrow::field("data", expected_struct, true)}); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { + // MAP value is nullable + auto nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), true)); + auto schema_nullable = arrow::schema({arrow::field("m", nullable_map)}); + std::map col_map = {{"m", 2}}; + + ASSERT_OK_AND_ASSIGN( + auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); + auto struct_type = physical->field(0)->type(); + ASSERT_TRUE(struct_type->field(0)->nullable()); + ASSERT_TRUE(struct_type->field(1)->nullable()); + ASSERT_TRUE(struct_type->field(2)->nullable()); + + // MAP value is non-nullable + auto non_nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), false)); + auto schema_non_nullable = arrow::schema({arrow::field("m", non_nullable_map)}); + + ASSERT_OK_AND_ASSIGN(auto physical2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema_non_nullable, col_map)); + auto struct_type2 = physical2->field(0)->type(); + ASSERT_FALSE(struct_type2->field(1)->nullable()); + ASSERT_FALSE(struct_type2->field(2)->nullable()); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaPreservesFieldMetadata) { + auto metadata = std::make_shared(); + metadata->Append("paimon.field.id", "7"); + metadata->Append("description", "original map field"); + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + auto schema = arrow::schema({arrow::field("m", map_type, false, metadata)}); + std::map col_map = {{"m", 2}}; + + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map)); + + ASSERT_FALSE(physical_schema->field(0)->nullable()); + ASSERT_TRUE(physical_schema->field(0)->metadata()->Equals(*metadata)); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + }); + + std::map empty_map; + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map)); + ASSERT_TRUE(physical_schema->Equals(schema)); +} + +TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithOverflow) { + auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + arrow::int64(), /*physical_col_ids=*/{3, 1}, /*value_nullable=*/false, + /*include_overflow=*/true); + + auto expected = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_1", arrow::int64(), false), + arrow::field("__col_3", arrow::int64(), false), + arrow::field("__overflow", + arrow::map(arrow::int32(), arrow::field("value", arrow::int64(), false)), + true), + }); + ASSERT_TRUE(actual->Equals(*expected)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); +} + +TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithoutOverflow) { + auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + arrow::utf8(), /*physical_col_ids=*/{3}, /*value_nullable=*/true, + /*include_overflow=*/false); + + auto expected = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_3", arrow::utf8(), true), + }); + ASSERT_TRUE(actual->Equals(*expected)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); +} + +// ---- BuildColumnToNumColumns ---- + +TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.shared-shredding.max-columns", "128"}, + {"fields.metrics.map.shared-shredding.max-columns", "64"}})); + + std::vector shredding_field_names = {"tags", "metrics"}; + ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( + shredding_field_names, options)); + + ASSERT_EQ(result.size(), 2); + ASSERT_EQ(result.at("tags"), 128); + ASSERT_EQ(result.at("metrics"), 64); +} + +TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumnsDefault) { + // No explicit max-columns config -> default 256 + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + std::vector shredding_field_names = {"tags"}; + ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( + shredding_field_names, options)); + ASSERT_EQ(result.at("tags"), 256); +} + +// ---- SerializeMetadata / DeserializeMetadata roundtrip ---- + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripNoneCompression) { + MapSharedShreddingFieldMeta original; + original.name_to_id = {{"age", 0}, {"name", 1}}; + original.field_to_columns = {{0, {0}}, {1, {1, 2}}}; + original.overflow_field_set = {1, 5}; + original.num_columns = 3; + original.max_row_width = 2; + + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(original, "none", metadata.get())); + + // Verify raw KV strings + auto find_value = [&](const char* key) -> std::string { + int32_t idx = metadata->FindKey(key); + EXPECT_GE(idx, 0); + return metadata->value(idx); + }; + ASSERT_EQ(find_value(MapShreddingDefine::kStorageLayout), "shared-shredding"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kVersion), "1"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kNumColumns), "3"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kMaxRowWidth), "2"); + + std::string expected_dict = R"({"age":0,"name":1})"; + ASSERT_EQ(find_value(MapSharedShreddingDefine::kFieldDict), expected_dict); + // field_dict_original_size should be the length of the JSON string + std::string field_dict_original_size = + find_value(MapSharedShreddingDefine::kFieldDictOriginalSize); + ASSERT_EQ(field_dict_original_size, std::to_string(expected_dict.size())); + + std::string expected_field_to_columns = R"({"0":[0],"1":[1,2]})"; + ASSERT_EQ(find_value(MapSharedShreddingDefine::kFieldColumns), expected_field_to_columns); + + // overflow_set is a JSON array of sorted field_ids + ASSERT_EQ(find_value(MapSharedShreddingDefine::kOverflowSet), "[1,5]"); + + // Roundtrip verify + ASSERT_OK_AND_ASSIGN(auto deserialized, MapSharedShreddingUtils::DeserializeMetadata(metadata)); + ASSERT_EQ(deserialized, original); +} + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripCompression) { + MapSharedShreddingFieldMeta original; + original.name_to_id = {{"alpha", 0}, {"beta", 1}, {"gamma", 2}}; + original.field_to_columns = {{0, {0, 1, 2}}, {1, {3}}, {2, {4, 5}}}; + original.overflow_field_set = {2}; + original.num_columns = 6; + original.max_row_width = 3; + + auto verify_roundtrip = [&](const std::string& compression) { + auto metadata = std::make_shared(); + ASSERT_OK( + MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get())); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(metadata)); + ASSERT_EQ(deserialized, original); + }; + + verify_roundtrip("none"); + verify_roundtrip("lz4"); + verify_roundtrip("zstd"); +} + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripEmptyData) { + MapSharedShreddingFieldMeta original; + + auto verify_roundtrip = [&](const std::string& compression) { + auto metadata = std::make_shared(); + ASSERT_OK( + MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get())); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(metadata)); + ASSERT_EQ(deserialized, original); + }; + + verify_roundtrip("none"); + verify_roundtrip("lz4"); + verify_roundtrip("zstd"); +} + +TEST(MapSharedShreddingUtilsTest, DeserializeLegacyMetadataWithoutCompression) { + MapSharedShreddingFieldMeta original; + original.name_to_id = {{"alpha", 0}, {"beta", 1}}; + original.field_to_columns = {{0, {0}}, {1, {1}}}; + original.overflow_field_set = {1}; + original.num_columns = 2; + original.max_row_width = 2; + + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata( + original, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); + ASSERT_TRUE(metadata->Delete(MapSharedShreddingDefine::kFieldDictCompression).ok()); + + ASSERT_OK_AND_ASSIGN(auto deserialized, MapSharedShreddingUtils::DeserializeMetadata(metadata)); + ASSERT_EQ(deserialized, original); +} + +// ---- DeserializeMetadata error cases ---- + +TEST(MapSharedShreddingUtilsTest, DeserializeMetadataErrors) { + const std::string layout_error = "metadata is null or storage layout is not shared-shredding"; + // nullptr + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(nullptr), layout_error); + // missing storage layout + { + auto metadata = std::make_shared(); + metadata->Append("some_key", "some_value"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), layout_error); + } + // wrong storage layout + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, "default"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), layout_error); + } + // missing version + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), + "missing shredding metadata key: paimon.map.shared-shredding.version"); + } + // wrong version + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + metadata->Append(MapSharedShreddingDefine::kVersion, "999"); + metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2"); + metadata->Append(MapSharedShreddingDefine::kFieldDict, "{}"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), + "unsupported shared-shredding metadata version: 999"); + } + // missing field_dict + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + metadata->Append(MapSharedShreddingDefine::kVersion, "1"); + metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2"); + ASSERT_NOK_WITH_MSG( + MapSharedShreddingUtils::DeserializeMetadata(metadata), + "missing shredding metadata key: paimon.map.shared-shredding.field-dict"); + } +} + +// ---- HasShreddingMetadata ---- + +TEST(MapSharedShreddingUtilsTest, HasShreddingMetadata) { + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(nullptr)); + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, "default"); + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } + { + auto metadata = std::make_shared(); + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } +} + +// ---- PhysicalColumnName ---- + +TEST(MapSharedShreddingUtilsTest, PhysicalColumnName) { + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(0), "__col_0"); + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(1), "__col_1"); + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(99), "__col_99"); +} + +// ---- GetPhysicalColumnIndices ---- + +// Normal: single physical column per field +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesSingleColumn) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"age", 0}, {"name", 1}}; + meta.field_to_columns = {{0, {2}}, {1, {5}}}; + + ASSERT_OK_AND_ASSIGN(auto cols_age, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "age")); + ASSERT_EQ(cols_age, (std::vector{2})); + + ASSERT_OK_AND_ASSIGN(auto cols_name, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "name")); + ASSERT_EQ(cols_name, (std::vector{5})); +} + +// Normal: multiple physical columns for one field +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesMultipleColumns) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"tags", 0}}; + meta.field_to_columns = {{0, {0, 3, 7}}}; + + ASSERT_OK_AND_ASSIGN(auto cols, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "tags")); + ASSERT_EQ(cols, (std::vector{0, 3, 7})); +} + +// Normal: many fields each mapping to different physical columns +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesMultipleFields) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {2, 3, 4}}, {2, {5}}}; + + ASSERT_OK_AND_ASSIGN(auto cols_a, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "a")); + ASSERT_EQ(cols_a, (std::vector{0, 1})); + + ASSERT_OK_AND_ASSIGN(auto cols_b, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "b")); + ASSERT_EQ(cols_b, (std::vector{2, 3, 4})); + + ASSERT_OK_AND_ASSIGN(auto cols_c, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "c")); + ASSERT_EQ(cols_c, (std::vector{5})); +} + +// Error: field name not found in name_to_id +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesFieldNotFound) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"age", 0}}; + meta.field_to_columns = {{0, {1}}}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "nonexistent"), + "cannot find field nonexistent in map shared shredding meta"); +} + +// Error: field name not found in empty meta +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesEmptyMeta) { + MapSharedShreddingFieldMeta meta; + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "any"), + "cannot find field any in map shared shredding meta"); +} + +// Error: field id exists in name_to_id but is missing from field_to_columns +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesFieldIdMissingInFieldToColumns) { + MapSharedShreddingFieldMeta meta; + // "score" -> field_id 42, but field_to_columns has no entry for 42 + meta.name_to_id = {{"score", 42}}; + meta.field_to_columns = {}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "score"), + "cannot find field id 42 in field_to_columns in map shared shredding meta"); +} + +TEST(MapSharedShreddingUtilsTest, IsOverflowField) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.overflow_field_set = {0, 2}; + + ASSERT_OK_AND_ASSIGN(bool a_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "a")); + ASSERT_TRUE(a_overflow); + + ASSERT_OK_AND_ASSIGN(bool b_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "b")); + ASSERT_FALSE(b_overflow); + + ASSERT_OK_AND_ASSIGN(bool c_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "c")); + ASSERT_TRUE(c_overflow); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::IsOverflowField(meta, "missing"), + "cannot find field missing in map shared shredding meta"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp new file mode 100644 index 00000000..04c821a7 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h" + +#include + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" + +namespace paimon { + +Result> +MapSharedShreddingWritePlanFactory::Create(const CoreOptions& options, + const std::shared_ptr& write_schema, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::vector shredding_fields, + MapSharedShreddingUtils::DetectShreddingColumns(write_schema, options)); + std::map field_to_max_columns; + PAIMON_ASSIGN_OR_RAISE(field_to_max_columns, MapSharedShreddingUtils::BuildColumnToNumColumns( + shredding_fields, options)); + return std::shared_ptr( + new MapSharedShreddingWritePlanFactory(options, write_schema, field_to_max_columns, pool)); +} + +MapSharedShreddingWritePlanFactory::MapSharedShreddingWritePlanFactory( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::map& field_to_max_columns, + const std::shared_ptr& pool) + : options_(options), + write_schema_(write_schema), + field_to_max_columns_(field_to_max_columns), + context_(std::make_shared(field_to_max_columns)), + pool_(pool) {} + +bool MapSharedShreddingWritePlanFactory::ShouldCreateWritePlan() const { + return !field_to_max_columns_.empty(); +} + +bool MapSharedShreddingWritePlanFactory::ShouldInferWritePlan() const { + return false; +} + +int32_t MapSharedShreddingWritePlanFactory::InferBufferRowCount() const { + return 0; +} + +Result> +MapSharedShreddingWritePlanFactory::CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) { + if (!ShouldCreateWritePlan()) { + return Status::Invalid("MAP shared-shredding write plan is not active."); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr converter, + MapSharedShreddingBatchConverter::Create(write_schema_, context_, options_, pool_)); + return std::shared_ptr(std::move(converter)); +} + +ShreddingWritePlanFactory::MetadataFinalizer +MapSharedShreddingWritePlanFactory::CreateMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression) const { + // The converter is created by CreateConverter above; the concrete type is guaranteed. + auto map_converter = std::static_pointer_cast(converter); + return MapSharedShreddingUtils::BuildMetadataFinalizer(map_converter, compression, + map_converter->GetPhysicalSchema()); +} + +Status MapSharedShreddingWritePlanFactory::OnFileCompleted( + const std::shared_ptr& converter) { + auto map_converter = std::dynamic_pointer_cast(converter); + if (map_converter == nullptr) { + return Status::Invalid("Unexpected converter for MAP shared-shredding."); + } + std::vector> completed_stats; + for (const std::string& field_name : map_converter->GetShreddingColumnNames()) { + PAIMON_ASSIGN_OR_RAISE(int32_t max_row_width, map_converter->GetMaxRowWidth(field_name)); + completed_stats.emplace_back(field_name, max_row_width); + } + for (const auto& [field_name, max_row_width] : completed_stats) { + context_->ReportFileStats(field_name, max_row_width); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h new file mode 100644 index 00000000..cb7afb1c --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/core/core_options.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +class MapSharedShreddingContext; + +/// Detects configured MAP shared-shredding fields and owns their cross-file adaptive-K context. +/// The write plan is never inferred from samples; per-file field metadata is persisted into the +/// file footer by the metadata finalizer. +class MapSharedShreddingWritePlanFactory : public ShreddingWritePlanFactory { + public: + static Result> Create( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool); + + bool ShouldCreateWritePlan() const override; + + bool ShouldInferWritePlan() const override; + + int32_t InferBufferRowCount() const override; + + Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) override; + + MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression) const override; + + Status OnFileCompleted(const std::shared_ptr& converter) override; + + private: + MapSharedShreddingWritePlanFactory(const CoreOptions& options, + const std::shared_ptr& write_schema, + const std::map& field_to_max_columns, + const std::shared_ptr& pool); + + CoreOptions options_; + std::shared_ptr write_schema_; + std::map field_to_max_columns_; + std::shared_ptr context_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shredding_defs.h b/src/paimon/common/data/shredding/map_shredding_defs.h new file mode 100644 index 00000000..44348b8f --- /dev/null +++ b/src/paimon/common/data/shredding/map_shredding_defs.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" + +namespace paimon { + +/// Constants for MAP storage layout marker. +struct MapShreddingDefine { + /// Marker key indicating this file uses a specific MAP storage layout. + static constexpr const char* kStorageLayout = "paimon.map.storage-layout"; + /// Value for kStorageLayout when using shared-shredding layout. + static constexpr const char* kStorageLayoutSharedShredding = "shared-shredding"; +}; + +/// Constants for the shared-shredding MAP storage layout. +/// Includes file footer meta keys and physical sub-column names. +struct MapSharedShreddingDefine { + // ---- File footer meta keys ---- + + /// Version of the shared-shredding meta format. + static constexpr const char* kVersion = "paimon.map.shared-shredding.version"; + /// Current meta format version. + static constexpr int32_t kCurrentVersion = 1; + /// JSON-encoded field name <-> field id dictionary (may be compressed). + static constexpr const char* kFieldDict = "paimon.map.shared-shredding.field-dict"; + /// Compression codec used by field_dict. Missing in legacy files, which default to zstd. + static constexpr const char* kFieldDictCompression = + "paimon.map.shared-shredding.field-dict-compression"; + /// Original (uncompressed) size of field_dict value. + static constexpr const char* kFieldDictOriginalSize = + "paimon.map.shared-shredding.field-dict-original-size"; + /// JSON-encoded field_id -> set of physical column indices. + static constexpr const char* kFieldColumns = "paimon.map.shared-shredding.field-columns"; + /// JSON-encoded set of field_ids that ever spilled into __overflow. + static constexpr const char* kOverflowSet = "paimon.map.shared-shredding.overflow-set"; + /// The number of physical columns K used in this file. + static constexpr const char* kNumColumns = "paimon.map.shared-shredding.num-columns"; + /// The maximum row width observed in this file. + static constexpr const char* kMaxRowWidth = "paimon.map.shared-shredding.max-row-width"; + + // ---- Physical sub-column names ---- + + /// Per-row field mapping column name. + static constexpr const char* kFieldMapping = "__field_mapping"; + /// Overflow column name. + static constexpr const char* kOverflow = "__overflow"; + + /// Default compression codec for field_dict serialization. + static constexpr const char* kDefaultDictCompression = "zstd"; + + /// Returns the name of the i-th physical column: "__col_0", "__col_1", etc. + static std::string PhysicalColumnName(int32_t index) { + return "__col_" + std::to_string(index); + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..d8057442 --- /dev/null +++ b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that keeps the input field order and maps it to columns 0..K-1. +class PlainMapSharedShreddingColumnAllocator : public MapSharedShreddingColumnAllocator { + public: + explicit PlainMapSharedShreddingColumnAllocator(int32_t num_columns) + : MapSharedShreddingColumnAllocator(num_columns) {} + + RowAllocation AllocateRow(const std::vector& field_ids) override { + RowAllocation allocation = AllocateLeadingColumns(field_ids); + CommitRow(allocation, field_ids); + return allocation; + } + + protected: + RowAllocation AllocateLeadingColumns(const std::vector& field_ids) const { + RowAllocation allocation; + allocation.col_to_field.assign(num_columns_, -1); + for (size_t i = 0; i < field_ids.size(); ++i) { + int32_t field_id = field_ids[i]; + if (i < static_cast(num_columns_)) { + allocation.col_to_field[static_cast(i)] = field_id; + } else { + allocation.overflow_fields.push_back(field_id); + } + } + return allocation; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..e7197392 --- /dev/null +++ b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(PlainMapSharedShreddingColumnAllocatorTest, BasicAllocation) { + PlainMapSharedShreddingColumnAllocator allocator(3); + // 2 fields, K=3 -> all fit, no overflow + RowAllocation result = allocator.AllocateRow({10, 20}); + ExpectAllocation(result, {10, 20, -1}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, ExactlyKFields) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation result = allocator.AllocateRow({0, 1, 2}); + ExpectAllocation(result, {0, 1, 2}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, OverflowWhenExceedK) { + PlainMapSharedShreddingColumnAllocator allocator(2); + + RowAllocation result = allocator.AllocateRow({10, 20, 30, 40}); + ExpectAllocation(result, {10, 20}, {30, 40}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, EmptyRow) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation result = allocator.AllocateRow({}); + ExpectAllocation(result, {-1, -1, -1}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, MaxRowWidthTracked) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({1, 2}); + ASSERT_EQ(2, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1, 2, 3, 4, 5}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, FieldToColumnsAccumulated) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({10, 20, 30}); + allocator.AllocateRow({20, 40}); + + const auto& field_to_cols = allocator.GetFieldToColumns(); + // field 10 -> {0} + ASSERT_EQ(std::set({0}), field_to_cols.at(10)); + // field 20 -> {1, 0} (col 1 in row 0, col 0 in row 1) + ASSERT_EQ(std::set({0, 1}), field_to_cols.at(20)); + // field 30 -> {2} + ASSERT_EQ(std::set({2}), field_to_cols.at(30)); + // field 40 -> {1} + ASSERT_EQ(std::set({1}), field_to_cols.at(40)); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, OverflowFieldSetAccumulated) { + PlainMapSharedShreddingColumnAllocator allocator(2); + + allocator.AllocateRow({1, 2, 3}); // 3 overflows + allocator.AllocateRow({4, 5, 6, 7}); // 6, 7 overflow + + ASSERT_EQ((std::set{3, 6, 7}), allocator.GetOverflowFieldSet()); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, SingleColumnAllocator) { + PlainMapSharedShreddingColumnAllocator allocator(1); + + RowAllocation result = allocator.AllocateRow({10, 20, 30}); + ExpectAllocation(result, {10}, {20, 30}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, UsesInputOrder) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({2, 0, 1}); + ExpectAllocation(row0, {2, 0, 1}, {}); + + RowAllocation row1 = allocator.AllocateRow({4, 3, 5, 6}); + ExpectAllocation(row1, {4, 3, 5}, {6}); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{1}), field_to_columns.at(0)); + ASSERT_EQ((std::set{2}), field_to_columns.at(1)); + ASSERT_EQ((std::set{0}), field_to_columns.at(2)); + ASSERT_EQ((std::set{1}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{2}), field_to_columns.at(5)); + ASSERT_EQ((std::set{6}), allocator.GetOverflowFieldSet()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..e26feede --- /dev/null +++ b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that sorts input fields and maps them to columns 0..K-1. +class SequentialMapSharedShreddingColumnAllocator : public PlainMapSharedShreddingColumnAllocator { + public: + explicit SequentialMapSharedShreddingColumnAllocator(int32_t num_columns) + : PlainMapSharedShreddingColumnAllocator(num_columns) {} + + RowAllocation AllocateRow(const std::vector& field_ids) override { + std::vector sorted_field_ids = field_ids; + std::sort(sorted_field_ids.begin(), sorted_field_ids.end()); + RowAllocation allocation = AllocateLeadingColumns(sorted_field_ids); + CommitRow(allocation, sorted_field_ids); + return allocation; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..f68ef0d7 --- /dev/null +++ b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(SequentialMapSharedShreddingColumnAllocatorTest, SortsAndUsesLeadingColumns) { + SequentialMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({1, 2}); + ExpectAllocation(row0, {1, 2, -1}, {}); + + RowAllocation row1 = allocator.AllocateRow({2, 3}); + ExpectAllocation(row1, {2, 3, -1}, {}); + + RowAllocation row2 = allocator.AllocateRow({7, 4, 6, 5}); + ExpectAllocation(row2, {4, 5, 6}, {7}); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{0}), field_to_columns.at(1)); + ASSERT_EQ((std::set{0, 1}), field_to_columns.at(2)); + ASSERT_EQ((std::set{1}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{1}), field_to_columns.at(5)); + ASSERT_EQ((std::set{2}), field_to_columns.at(6)); + ASSERT_EQ((std::set{7}), allocator.GetOverflowFieldSet()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/shredding_batch_converter.h b/src/paimon/common/data/shredding/shredding_batch_converter.h new file mode 100644 index 00000000..736ea4e2 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_batch_converter.h @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/c/abi.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +/// Converts logical write batches into a physical (shredded) file layout. Implemented by both +/// the VARIANT shredding and the MAP shared-shredding batch converters so that shredded file +/// writers can be composed generically. +class ShreddingBatchConverter { + public: + virtual ~ShreddingBatchConverter() = default; + + /// The physical file schema that converted batches conform to. + virtual const std::shared_ptr& GetPhysicalSchema() const = 0; + + /// Converts a logical batch into the physical layout. Consumes `logical_batch`. + virtual Result> Convert(::ArrowArray* logical_batch) = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_file_reader.cpp b/src/paimon/common/data/shredding/shredding_file_reader.cpp new file mode 100644 index 00000000..cb0eaa28 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_file_reader.cpp @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shredding_file_reader.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +ShreddingFileReader::ShreddingFileReader( + std::unique_ptr&& reader, + std::map>&& plans, + const std::shared_ptr& pool) + : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)), plans_(std::move(plans)) {} + +Result> ShreddingFileReader::GetFileSchema() const { + return reader_->GetFileSchema(); +} + +Status ShreddingFileReader::SetReadSchema(::ArrowSchema* read_schema, + const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (!read_schema) { + return Status::Invalid("invalid read schema in ShreddingFileReader, cannot be null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_read_schema, + arrow::ImportSchema(read_schema)); + arrow::FieldVector resolved_fields = logical_read_schema->fields(); + bool any_resolved = false; + for (auto& resolved_field : resolved_fields) { + auto it = plans_.find(resolved_field->name()); + if (it == plans_.end()) { + continue; + } + // Push the physical (possibly pruned) subtree down so the inner reader materializes it. + resolved_field = it->second->PhysicalField(); + any_resolved = true; + } + if (!any_resolved) { + return Status::Invalid("no planned shredded columns exist in the read schema"); + } + auto resolved_schema = arrow::schema(resolved_fields, logical_read_schema->metadata()); + auto c_resolved_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*resolved_schema, c_resolved_schema.get())); + return reader_->SetReadSchema(c_resolved_schema.get(), predicate, selection_bitmap); +} + +Result ShreddingFileReader::NextBatch() { + return Status::Invalid( + "paimon inner reader ShreddingFileReader should use NextBatchWithBitmap"); +} + +Result ShreddingFileReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + reader_->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return batch_with_bitmap; + } + + auto& [batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + if (arrow_array->type_id() != arrow::Type::STRUCT) { + return Status::Invalid("cannot cast batch to StructArray in ShreddingFileReader"); + } + auto struct_array = std::static_pointer_cast(arrow_array); + + arrow::ArrayVector resolved_arrays = struct_array->fields(); + arrow::FieldVector resolved_fields = struct_array->struct_type()->fields(); + for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { + const auto& physical_field = struct_array->struct_type()->field(field_idx); + auto it = plans_.find(physical_field->name()); + if (it == plans_.end()) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr logical_array, + it->second->Assemble(struct_array->field(field_idx), arrow_pool_.get())); + resolved_arrays[field_idx] = logical_array; + resolved_fields[field_idx] = it->second->LogicalField(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_struct_array, + arrow::StructArray::Make(resolved_arrays, resolved_fields)); + auto new_c_array = std::make_unique(); + auto new_c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*new_struct_array, new_c_array.get(), new_c_schema.get())); + batch = std::make_pair(std::move(new_c_array), std::move(new_c_schema)); + return batch_with_bitmap; +} + +std::shared_ptr ShreddingFileReader::GetReaderMetrics() const { + return reader_->GetReaderMetrics(); +} + +void ShreddingFileReader::Close() { + reader_->Close(); +} + +Result ShreddingFileReader::GetPreviousBatchFileRowId(uint64_t batch_row_id) const { + return reader_->GetPreviousBatchFileRowId(batch_row_id); +} + +Result ShreddingFileReader::GetNumberOfRows() const { + return reader_->GetNumberOfRows(); +} + +bool ShreddingFileReader::SupportPreciseBitmapSelection() const { + return reader_->SupportPreciseBitmapSelection(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_file_reader.h b/src/paimon/common/data/shredding/shredding_file_reader.h new file mode 100644 index 00000000..f6ab3575 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_file_reader.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/shredding/shredding_read_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" + +namespace paimon { + +/// A file batch reader wrapper that pushes per-column physical (shredded, possibly pruned) +/// subtrees down to the inner reader and assembles the physical batches back into the logical +/// columns as planned by `ShreddingColumnReadPlan`s. +class ShreddingFileReader : public FileBatchReader { + public: + ShreddingFileReader(std::unique_ptr&& reader, + std::map>&& plans, + const std::shared_ptr& pool); + + Result> GetFileSchema() const override; + + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; + + Result GetNumberOfRows() const override; + + bool SupportPreciseBitmapSelection() const override; + + private: + std::shared_ptr arrow_pool_; + std::unique_ptr reader_; + std::map> plans_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_read_plan.h b/src/paimon/common/data/shredding/shredding_read_plan.h new file mode 100644 index 00000000..a9338f0b --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_read_plan.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Field; +class MemoryPool; +} // namespace arrow + +namespace paimon { + +/// A per-column read plan translating between a column's logical shape and the physical +/// (shredded, possibly pruned) shape stored in one file: the physical field is pushed down to +/// the format reader and the physical batches are assembled back into logical arrays. +class ShreddingColumnReadPlan { + public: + virtual ~ShreddingColumnReadPlan() = default; + + /// The logical field restored on the assembled output. + virtual const std::shared_ptr& LogicalField() const = 0; + + /// The physical file field (possibly a pruned subtree) to read from the file. + virtual const std::shared_ptr& PhysicalField() const = 0; + + /// Assembles the physical column array back into the logical column array. + virtual Result> Assemble( + const std::shared_ptr& physical, arrow::MemoryPool* pool) const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp new file mode 100644 index 00000000..ace79875 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" + +#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" +#include "paimon/core/core_options.h" + +namespace paimon { + +Result> ShreddingWritePlanFactories::SelectActive( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool) { + std::shared_ptr active_factory; + + auto variant_factory = VariantShreddingWritePlanFactory::Create(options, write_schema, pool); + if (variant_factory->ShouldCreateWritePlan()) { + active_factory = std::move(variant_factory); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr map_factory, + MapSharedShreddingWritePlanFactory::Create(options, write_schema, pool)); + if (map_factory->ShouldCreateWritePlan()) { + if (active_factory != nullptr) { + return Status::NotImplemented( + "Composing multiple active shredding write plans is not supported."); + } + active_factory = std::move(map_factory); + } + + return active_factory; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.h b/src/paimon/common/data/shredding/shredding_write_plan_factories.h new file mode 100644 index 00000000..c5cfb495 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +/// Composes the known shredding write-plan factories (MAP shared-shredding and VARIANT +/// shredding) and selects the one active for a write schema. +class ShreddingWritePlanFactories { + public: + /// Returns the single active write-plan factory for the write, or nullptr when no shredding + /// applies. Each concrete factory detects whether it is active and owns its internal state. + static Result> SelectActive( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factory.h b/src/paimon/common/data/shredding/shredding_write_plan_factory.h new file mode 100644 index 00000000..ed05a209 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factory.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_batch_converter.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Decides whether write batches must be rewritten into a physical (shredded) layout before +/// they reach the format writer, and creates the per-file batch converter -- either immediately +/// from configuration or, for inference, from sampled logical batches buffered by the writer. +class ShreddingWritePlanFactory { + public: + /// Finalizes per-file shredding metadata; the returned schema (or nullptr) is persisted into + /// the file footer right before the file is finished. + using MetadataFinalizer = std::function>()>; + + virtual ~ShreddingWritePlanFactory() = default; + + /// Whether a write plan (immediate or inferred) applies to the write schema. + virtual bool ShouldCreateWritePlan() const = 0; + + /// Whether the write plan must be inferred from sampled rows instead of configuration. + virtual bool ShouldInferWritePlan() const = 0; + + /// The number of rows buffered per file to sample the inferred write plan from. + virtual int32_t InferBufferRowCount() const = 0; + + /// Creates the per-file batch converter. `sample_batches` holds the logical batches sampled + /// for inference and is empty when the plan comes from configuration. Returns nullptr when + /// no conversion is useful for this file (the file is written with the logical schema). + virtual Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) = 0; + + /// The per-file metadata finalizer persisted into the file footer, or nullptr when the + /// physical schema is self-describing (as it is for VARIANT shredding). + virtual MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression) const = 0; + + /// Advances rolling-writer-scoped state after the file and its output stream have been + /// closed successfully. Failed files must never affect the next file's write plan. + virtual Status OnFileCompleted(const std::shared_ptr& converter) = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant.cpp b/src/paimon/common/data/variant/generic_variant.cpp new file mode 100644 index 00000000..34b74c0b --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant.cpp @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/generic_variant.h" + +#include +#include +#include + +#include "arrow/util/base64.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_json_utils.h" + +namespace paimon { + +namespace { + +Status ToJsonImpl(std::string_view value, std::string_view metadata, int32_t pos, + const std::string& zone_id, std::string* out) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, VariantBinaryUtil::GetType(value, pos)); + switch (type) { + case VariantValueType::kObject: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(value, pos)); + out->push_back('{'); + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, + VariantBinaryUtil::ReadUnsigned( + value, info.id_start + info.id_size * i, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, value.size())); + if (i != 0) { + out->push_back(','); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + VariantJsonUtils::AppendEscapedJson(key, out); + out->push_back(':'); + PAIMON_RETURN_NOT_OK(ToJsonImpl(value, metadata, element_pos, zone_id, out)); + } + out->push_back('}'); + return Status::OK(); + } + case VariantValueType::kArray: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(value, pos)); + out->push_back('['); + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, value.size())); + if (i != 0) { + out->push_back(','); + } + PAIMON_RETURN_NOT_OK(ToJsonImpl(value, metadata, element_pos, zone_id, out)); + } + out->push_back(']'); + return Status::OK(); + } + case VariantValueType::kNull: + out->append("null"); + return Status::OK(); + case VariantValueType::kBoolean: { + PAIMON_ASSIGN_OR_RAISE(bool b, VariantBinaryUtil::GetBoolean(value, pos)); + out->append(b ? "true" : "false"); + return Status::OK(); + } + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t l, VariantBinaryUtil::GetLong(value, pos)); + out->append(std::to_string(l)); + return Status::OK(); + } + case VariantValueType::kString: { + PAIMON_ASSIGN_OR_RAISE(std::string_view s, VariantBinaryUtil::GetString(value, pos)); + VariantJsonUtils::AppendEscapedJson(s, out); + return Status::OK(); + } + case VariantValueType::kDouble: { + PAIMON_ASSIGN_OR_RAISE(double d, VariantBinaryUtil::GetDouble(value, pos)); + std::string repr = VariantJsonUtils::JavaDoubleToString(d); + if (std::isfinite(d)) { + out->append(repr); + } else { + out->push_back('"'); + out->append(repr); + out->push_back('"'); + } + return Status::OK(); + } + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal d, VariantBinaryUtil::GetDecimal(value, pos)); + out->append(d.ToPlainString()); + return Status::OK(); + } + case VariantValueType::kDate: { + PAIMON_ASSIGN_OR_RAISE(int64_t days, VariantBinaryUtil::GetLong(value, pos)); + out->push_back('"'); + out->append(VariantJsonUtils::DateToString(static_cast(days))); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kTimestamp: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, VariantBinaryUtil::GetLong(value, pos)); + PAIMON_ASSIGN_OR_RAISE(int32_t offset_seconds, + VariantJsonUtils::GetZoneOffsetSeconds(zone_id, micros)); + out->push_back('"'); + out->append(VariantJsonUtils::TimestampToString(micros, offset_seconds, true)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kTimestampNtz: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, VariantBinaryUtil::GetLong(value, pos)); + out->push_back('"'); + out->append(VariantJsonUtils::TimestampToString(micros, 0, false)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kFloat: { + PAIMON_ASSIGN_OR_RAISE(float f, VariantBinaryUtil::GetFloat(value, pos)); + std::string repr = VariantJsonUtils::JavaFloatToString(f); + if (std::isfinite(f)) { + out->append(repr); + } else { + out->push_back('"'); + out->append(repr); + out->push_back('"'); + } + return Status::OK(); + } + case VariantValueType::kBinary: { + PAIMON_ASSIGN_OR_RAISE(std::string_view binary, + VariantBinaryUtil::GetBinary(value, pos)); + out->push_back('"'); + out->append(arrow::util::base64_encode(binary)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kUuid: { + PAIMON_ASSIGN_OR_RAISE(std::string_view uuid, VariantBinaryUtil::GetUuid(value, pos)); + out->push_back('"'); + out->append(VariantBinaryUtil::UuidToString(uuid)); + out->push_back('"'); + return Status::OK(); + } + } + return VariantBinaryUtil::MalformedVariant("unknown variant value type in JSON rendering"); +} + +} // namespace + +GenericVariant::GenericVariant(std::shared_ptr value, std::shared_ptr metadata, + int32_t pos) + : value_(std::move(value)), metadata_(std::move(metadata)), pos_(pos) {} + +Result> GenericVariant::Create(std::shared_ptr value, + std::shared_ptr metadata) { + if (!value || !metadata) { + return Status::Invalid("variant value and metadata must not be null"); + } + // There is currently only one allowed version. + if (metadata->size() < 1 || (static_cast((*metadata)[0]) & + VariantDefs::kVersionMask) != VariantDefs::kVersion) { + return VariantBinaryUtil::MalformedVariant("unsupported variant metadata version"); + } + // Don't attempt to use a Variant larger than 128 MiB. We'll never produce one, and it risks + // memory instability. + if (metadata->size() > static_cast(VariantDefs::kSizeLimit) || + value->size() > static_cast(VariantDefs::kSizeLimit)) { + return VariantBinaryUtil::VariantConstructorSizeLimit(); + } + return std::shared_ptr( + new GenericVariant(std::move(value), std::move(metadata), 0)); +} + +Result> GenericVariant::Create( + std::string_view value, std::string_view metadata, const std::shared_ptr& pool) { + // Reject over-limit inputs before allocating and copying them. + if (value.size() > static_cast(VariantDefs::kSizeLimit) || + metadata.size() > static_cast(VariantDefs::kSizeLimit)) { + return VariantBinaryUtil::VariantConstructorSizeLimit(); + } + std::shared_ptr value_bytes = Bytes::AllocateBytes(value.size(), pool.get()); + if (!value.empty()) { + std::memcpy(value_bytes->data(), value.data(), value.size()); + } + std::shared_ptr metadata_bytes = Bytes::AllocateBytes(metadata.size(), pool.get()); + if (!metadata.empty()) { + // An empty (malformed) metadata view may carry a null data pointer; the size check + // keeps memcpy away from it. + std::memcpy(metadata_bytes->data(), metadata.data(), metadata.size()); + } + return Create(std::move(value_bytes), std::move(metadata_bytes)); +} + +Result> GenericVariant::FromJson( + std::string_view json, const std::shared_ptr& pool) { + return VariantBuilder::ParseJson(json, /*allow_duplicate_keys=*/false, pool); +} + +Result GenericVariant::Value() const { + std::string_view raw = RawValue(); + if (pos_ == 0) { + return raw; + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, VariantBinaryUtil::ValueSize(raw, pos_)); + PAIMON_RETURN_NOT_OK( + VariantBinaryUtil::CheckIndex(pos_ + size - 1, static_cast(raw.size()))); + return raw.substr(pos_, size); +} + +std::string_view GenericVariant::RawValue() const { + return {value_->data(), value_->size()}; +} + +std::string_view GenericVariant::Metadata() const { + return {metadata_->data(), metadata_->size()}; +} + +int64_t GenericVariant::SizeInBytes() const { + return static_cast(value_->size()) + static_cast(metadata_->size()); +} + +Result GenericVariant::ToJson(const std::string& zone_id) const { + std::string result; + PAIMON_RETURN_NOT_OK(ToJsonImpl(RawValue(), Metadata(), pos_, zone_id, &result)); + return result; +} + +Result GenericVariant::GetType() const { + return VariantBinaryUtil::GetType(RawValue(), pos_); +} + +Result GenericVariant::GetTypeInfo() const { + return VariantBinaryUtil::GetTypeInfo(RawValue(), pos_); +} + +Result GenericVariant::GetBoolean() const { + return VariantBinaryUtil::GetBoolean(RawValue(), pos_); +} + +Result GenericVariant::GetLong() const { + return VariantBinaryUtil::GetLong(RawValue(), pos_); +} + +Result GenericVariant::GetDouble() const { + return VariantBinaryUtil::GetDouble(RawValue(), pos_); +} + +Result GenericVariant::GetDecimal() const { + return VariantBinaryUtil::GetDecimal(RawValue(), pos_); +} + +Result GenericVariant::GetFloat() const { + return VariantBinaryUtil::GetFloat(RawValue(), pos_); +} + +Result GenericVariant::GetBinary() const { + return VariantBinaryUtil::GetBinary(RawValue(), pos_); +} + +Result GenericVariant::GetString() const { + return VariantBinaryUtil::GetString(RawValue(), pos_); +} + +Result GenericVariant::GetUuid() const { + return VariantBinaryUtil::GetUuid(RawValue(), pos_); +} + +Result GenericVariant::ObjectSize() const { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(RawValue(), pos_)); + return info.num_elements; +} + +std::shared_ptr GenericVariant::SubVariant(int32_t pos) const { + return std::shared_ptr(new GenericVariant(value_, metadata_, pos)); +} + +Result> GenericVariant::GetFieldByKey(std::string_view key) const { + std::string_view raw = RawValue(); + std::string_view metadata = Metadata(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + // Use linear search for a short list. Switch to binary search when the length reaches + // `kBinarySearchThreshold`. + if (info.num_elements < VariantDefs::kBinarySearchThreshold) { + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t id, VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * i, + info.id_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view field_key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + if (field_key == key) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(raw, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, raw.size())); + return SubVariant(element_pos); + } + } + } else { + int32_t low = 0; + int32_t high = info.num_elements - 1; + while (low <= high) { + // Use an unsigned shift to compute the middle of `low` and `high`, which properly + // handles the case where `low + high` overflows int32. + auto mid = static_cast( + (static_cast(low) + static_cast(high)) >> 1); + PAIMON_ASSIGN_OR_RAISE( + int32_t id, VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * mid, + info.id_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view field_key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + int32_t cmp = field_key.compare(key); + if (cmp < 0) { + low = mid + 1; + } else if (cmp > 0) { + high = mid - 1; + } else { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(raw, info.offset_start + info.offset_size * mid, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, raw.size())); + return SubVariant(element_pos); + } + } + } + return std::shared_ptr(nullptr); +} + +Result> GenericVariant::GetFieldAtIndex( + int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return std::optional(std::nullopt); + } + PAIMON_ASSIGN_OR_RAISE( + int32_t id, + VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * index, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, VariantBinaryUtil::ReadUnsigned( + raw, info.offset_start + info.offset_size * index, info.offset_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view key, VariantBinaryUtil::GetMetadataKey(Metadata(), id)); + ObjectField field; + field.key = std::string(key); + PAIMON_ASSIGN_OR_RAISE(int32_t field_pos, VariantBinaryUtil::CheckedElementPos( + info.data_start, offset, raw.size())); + field.value = SubVariant(field_pos); + return std::optional(std::move(field)); +} + +Result GenericVariant::GetDictionaryIdAtIndex(int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return VariantBinaryUtil::MalformedVariant(fmt::format( + "object field index {} is out of bounds for {} fields", index, info.num_elements)); + } + return VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * index, info.id_size); +} + +Result GenericVariant::ArraySize() const { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(RawValue(), pos_)); + return info.num_elements; +} + +Result> GenericVariant::GetElementAtIndex(int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, VariantBinaryUtil::ReadUnsigned( + raw, info.offset_start + info.offset_size * index, info.offset_size)); + PAIMON_ASSIGN_OR_RAISE(int32_t element_pos, VariantBinaryUtil::CheckedElementPos( + info.data_start, offset, raw.size())); + return SubVariant(element_pos); +} + +bool GenericVariant::operator==(const GenericVariant& other) const { + return pos_ == other.pos_ && RawValue() == other.RawValue() && Metadata() == other.Metadata(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant.h b/src/paimon/common/data/variant/generic_variant.h new file mode 100644 index 00000000..f95d12f0 --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { + +/// A Variant represents a type that contains one of: 1) Primitive: A type and corresponding value +/// (e.g. INT, STRING); 2) Array: An ordered list of Variant values; 3) Object: An unordered +/// collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate +/// keys. +/// +/// A Variant is encoded with 2 binaries: the value and the metadata. The Variant Binary Encoding +/// allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently +/// queried by path. The design is intended to allow efficient access to nested data even in the +/// presence of very wide or deep structures. +class GenericVariant { + public: + /// A field of a variant object. + struct ObjectField { + std::string key; + std::shared_ptr value; + }; + + /// Creates a variant taking ownership of the given buffers. Fails with `MALFORMED_VARIANT` + /// if the metadata version is unsupported, or `VARIANT_CONSTRUCTOR_SIZE_LIMIT` if either + /// buffer exceeds the 128MiB size limit. + static Result> Create(std::shared_ptr value, + std::shared_ptr metadata); + + /// Creates a variant by copying the given buffers into `pool`. + static Result> Create(std::string_view value, + std::string_view metadata, + const std::shared_ptr& pool); + + /// Parses a JSON string as a variant (duplicate object keys are rejected). + static Result> FromJson( + std::string_view json, const std::shared_ptr& pool); + + /// The variant value binary. For a sub-variant (`Pos() != 0`), the view covers only the + /// sub-variant slice of the underlying buffer. + Result Value() const; + + /// The whole underlying value buffer, regardless of `Pos()`. + std::string_view RawValue() const; + + /// The variant metadata binary. + std::string_view Metadata() const; + + /// The variant value doesn't use the whole value binary, but starts from the `Pos()` index + /// and spans a size of `VariantBinaryUtil::ValueSize`. This design avoids frequent copies of + /// the value binary when reading a sub-variant in an array/object element. + int32_t Pos() const { + return pos_; + } + + /// The size of the variant in bytes (value size + metadata size). + int64_t SizeInBytes() const; + + /// Stringifies the variant in JSON format. `zone_id` controls the rendering of TIMESTAMP + /// values; supported forms are "UTC"/"Z"/"GMT" and fixed offsets such as "+08:00". + Result ToJson(const std::string& zone_id = "UTC") const; + + /// The value type of the variant. + Result GetType() const; + + /// The type info bits of the variant value header. + Result GetTypeInfo() const; + + Result GetBoolean() const; + Result GetLong() const; + Result GetDouble() const; + Result GetDecimal() const; + Result GetFloat() const; + Result GetBinary() const; + Result GetString() const; + /// The 16-byte big-endian UUID value. + Result GetUuid() const; + + /// The number of object fields in the variant. It is only legal to call it when `GetType()` + /// is `kObject`. + Result ObjectSize() const; + + /// Finds the field value whose key is equal to `key`. Returns nullptr if the key is not + /// found. It is only legal to call it when `GetType()` is `kObject`. The returned sub-variant + /// shares the underlying buffers with this variant. + Result> GetFieldByKey(std::string_view key) const; + + /// Gets the object field at the `index` slot. Returns nullopt if `index` is out of the bound + /// of `[0, ObjectSize())`. It is only legal to call it when `GetType()` is `kObject`. + Result> GetFieldAtIndex(int32_t index) const; + + /// Gets the metadata dictionary id for the object field at the `index` slot. It is only + /// legal to call it when `GetType()` is `kObject`. + Result GetDictionaryIdAtIndex(int32_t index) const; + + /// The number of array elements in the variant. It is only legal to call it when `GetType()` + /// is `kArray`. + Result ArraySize() const; + + /// Gets the array element at the `index` slot. Returns nullptr if `index` is out of the + /// bound of `[0, ArraySize())`. It is only legal to call it when `GetType()` is `kArray`. + Result> GetElementAtIndex(int32_t index) const; + + bool operator==(const GenericVariant& other) const; + + private: + GenericVariant(std::shared_ptr value, std::shared_ptr metadata, int32_t pos); + + std::shared_ptr SubVariant(int32_t pos) const; + + std::shared_ptr value_; + std::shared_ptr metadata_; + int32_t pos_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant_test.cpp b/src/paimon/common/data/variant/generic_variant_test.cpp new file mode 100644 index 00000000..11386a28 --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant_test.cpp @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/generic_variant.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class GenericVariantTest : public ::testing::Test { + public: + static std::string ToHex(std::string_view data) { + constexpr char kHexDigits[] = "0123456789abcdef"; + std::string result; + result.reserve(data.size() * 2); + for (char c : data) { + auto byte = static_cast(c); + result.push_back(kHexDigits[byte >> 4]); + result.push_back(kHexDigits[byte & 0xF]); + } + return result; + } + + std::shared_ptr FromJson(const std::string& json) { + auto result = GenericVariant::FromJson(json, pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + // Asserts that parsing `json` produces exactly the value/metadata binaries produced by the + // Java implementation (`GenericVariantBuilder`), and that rendering back to JSON matches the + // Java `toJson` output. + void CheckGolden(const std::string& json, const std::string& expected_value_hex, + const std::string& expected_metadata_hex, + const std::string& expected_to_json) { + auto variant = FromJson(json); + ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value()); + ASSERT_EQ(ToHex(value), expected_value_hex) << "value bytes mismatch for: " << json; + ASSERT_EQ(ToHex(variant->Metadata()), expected_metadata_hex) + << "metadata bytes mismatch for: " << json; + ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); + ASSERT_EQ(to_json, expected_to_json); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +// Golden binaries generated by the Java implementation (org.apache.paimon.data.variant); these +// pin the cross-implementation byte compatibility of the variant encoding. +TEST_F(GenericVariantTest, GoldenPrimitives) { + CheckGolden("null", "00", "010000", "null"); + CheckGolden("true", "04", "010000", "true"); + CheckGolden("false", "08", "010000", "false"); + CheckGolden("1", "0c01", "010000", "1"); + CheckGolden("-1", "0cff", "010000", "-1"); + CheckGolden("300", "102c01", "010000", "300"); + CheckGolden("100000", "14a0860100", "010000", "100000"); + CheckGolden("12345678901234", "18f22fce733a0b0000", "010000", "12345678901234"); + CheckGolden("1e40", "1ca55cc3f129633d48", "010000", "1.0E40"); + CheckGolden("1.0123456789012345678901234567890123456789", "1c240bf2619132f03f", "010000", + "1.0123456789012346"); + CheckGolden("2.5e-3", "1c7b14ae47e17a643f", "010000", "0.0025"); + CheckGolden("100.99", "200273270000", "010000", "100.99"); + CheckGolden("-0.5", "2001fbffffff", "010000", "-0.5"); + CheckGolden("0.0", "200100000000", "010000", "0"); + CheckGolden("12345678.90123", "2405cb04fb711f010000", "010000", "12345678.90123"); + CheckGolden("1234567890123456789.0123456789", "280a1581396eb1c9be46321be42700000000", "010000", + "1234567890123456789.0123456789"); + // An integer that overflows int64 is parsed as an exact decimal. + CheckGolden("123456789012345678901234567890", "2800d20a3f4eeee073c3f60fe98e01000000", "010000", + "123456789012345678901234567890"); +} + +TEST_F(GenericVariantTest, GoldenStrings) { + CheckGolden("\"Hello, World!\"", "3548656c6c6f2c20576f726c6421", "010000", "\"Hello, World!\""); + CheckGolden("\"\"", "01", "010000", "\"\""); + CheckGolden( + "\"This is a long string that definitely exceeds the sixty-three byte short string limit " + "...!\"", + "405a000000546869732069732061206c6f6e6720737472696e67207468617420646566696e6974656c792065" + "786365656473207468652073697874792d746872656520627974652073686f727420737472696e67206c696d" + "6974202e2e2e21", + "010000", + "\"This is a long string that definitely exceeds the sixty-three byte short string limit " + "...!\""); +} + +TEST_F(GenericVariantTest, GoldenContainers) { + CheckGolden("{}", "020000", "010000", "{}"); + CheckGolden("[]", "030000", "010000", "[]"); + CheckGolden(R"({"a": 1, "b": "hello"})", "020200010002080c011568656c6c6f", "01020001026162", + R"({"a":1,"b":"hello"})"); + CheckGolden(R"([1, "two", 3.5, null, true, {"k":[]}])", + "03060002060c0d0e160c010d74776f20012300000000040201000003030000", "010100016b", + R"([1,"two",3.5,null,true,{"k":[]}])"); +} + +TEST_F(GenericVariantTest, GoldenNested) { + CheckGolden( + "{\"object\":{\"name\":\"Apache Paimon\",\"age\":2,\"address\":{\"street\":\"Main " + "St\",\"city\":\"Hangzhou\"}},\"array\":[1,2,3,4,5],\"string\":\"Hello, " + "World!\",\"long\":12345678901234,\"double\":1." + "0123456789012345678901234567890123456789,\"decimal\":100.99,\"boolean1\":true," + "\"boolean2\":false,\"nullField\":null}", + "0209060b0c0a09080d000731696a635a516b00436c0203030201100e0028354170616368652050" + "61696d6f6e0c02020205040800111d4d61696e2053742148616e677a686f75030500020406080a" + "0c010c020c030c040c053548656c6c6f2c20576f726c642118f22fce733a0b00001c240bf26191" + "32f03f200273270000040800", + "010e00060a0d141a1e23292d333a424a536f626a6563746e616d6561676561646472657373737472656574" + "636974796172726179737472696e676c6f6e67646f75626c65646563696d616c626f6f6c65616e31626f6f" + "6c65616e326e756c6c4669656c64", + "{\"array\":[1,2,3,4,5],\"boolean1\":true,\"boolean2\":false,\"decimal\":100.99," + "\"double\":1.0123456789012346,\"long\":12345678901234,\"nullField\":null,\"object\":{" + "\"address\":{\"city\":\"Hangzhou\",\"street\":\"Main St\"},\"age\":2,\"name\":\"Apache " + "Paimon\"},\"string\":\"Hello, World!\"}"); +} + +TEST_F(GenericVariantTest, GoldenUnicodeEscape) { + CheckGolden(R"({"\u4e2d\u6587": "\u4f60\u597d\n\t\"quoted\""})", + "020100001141e4bda0e5a5bd0a092271756f74656422", "01010006e4b8ade69687", + "{\"中文\":\"你好\\n\\t\\\"quoted\\\"\"}"); +} + +TEST_F(GenericVariantTest, TypedAccessors) { + auto variant = FromJson(R"({"a": 1, "b": "hello"})"); + ASSERT_OK_AND_ASSIGN(VariantValueType type, variant->GetType()); + ASSERT_EQ(type, VariantValueType::kObject); + ASSERT_OK_AND_ASSIGN(int32_t object_size, variant->ObjectSize()); + ASSERT_EQ(object_size, 2); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr a, variant->GetFieldByKey("a")); + ASSERT_NE(a, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t a_value, a->GetLong()); + ASSERT_EQ(a_value, 1); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr b, variant->GetFieldByKey("b")); + ASSERT_NE(b, nullptr); + ASSERT_OK_AND_ASSIGN(std::string_view b_value, b->GetString()); + ASSERT_EQ(b_value, "hello"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, variant->GetFieldByKey("c")); + ASSERT_EQ(missing, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field0, variant->GetFieldAtIndex(0)); + ASSERT_TRUE(field0.has_value()); + ASSERT_EQ(field0->key, "a"); + ASSERT_OK_AND_ASSIGN(auto field_oob, variant->GetFieldAtIndex(2)); + ASSERT_FALSE(field_oob.has_value()); + + auto array = FromJson("[10, 20, 30]"); + ASSERT_OK_AND_ASSIGN(int32_t array_size, array->ArraySize()); + ASSERT_EQ(array_size, 3); + ASSERT_OK_AND_ASSIGN(std::shared_ptr elem, array->GetElementAtIndex(1)); + ASSERT_NE(elem, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t elem_value, elem->GetLong()); + ASSERT_EQ(elem_value, 20); + ASSERT_OK_AND_ASSIGN(std::shared_ptr elem_oob, array->GetElementAtIndex(3)); + ASSERT_EQ(elem_oob, nullptr); +} + +TEST_F(GenericVariantTest, ObjectBinarySearch) { + // More fields than kBinarySearchThreshold exercises the binary-search lookup. + std::string json = "{"; + for (int32_t i = 0; i < 40; ++i) { + if (i != 0) { + json += ","; + } + json += "\"key" + std::to_string(i) + "\":" + std::to_string(i); + } + json += "}"; + auto variant = FromJson(json); + for (int32_t i = 0; i < 40; ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + variant->GetFieldByKey("key" + std::to_string(i))); + ASSERT_NE(field, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t value, field->GetLong()); + ASSERT_EQ(value, i); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, variant->GetFieldByKey("key40")); + ASSERT_EQ(missing, nullptr); +} + +TEST_F(GenericVariantTest, DuplicateKeys) { + ASSERT_NOK(GenericVariant::FromJson("{\"a\": 1, \"a\": 2}", pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + VariantBuilder::ParseJson("{\"a\": 1, \"a\": 2}", /*allow_duplicate_keys=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); + ASSERT_EQ(to_json, "{\"a\":2}"); +} + +TEST_F(GenericVariantTest, CorruptedLayoutRejected) { + // Headers claiming a near-INT32_MAX element count must be rejected by the 64-bit layout + // bound instead of overflowing the 32-bit offset arithmetic. + std::string metadata; + metadata.push_back(static_cast(0x01)); + metadata.push_back(static_cast(0x00)); + { + // Object, large size, 4-byte ids and offsets, num_elements = INT32_MAX. + std::string value; + value.push_back(static_cast((0x1F << 2) | 0x02)); + value.append({static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0x7F)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + ASSERT_NOK(variant->ToJson()); + } + { + // Array, large size, 4-byte offsets, num_elements = INT32_MAX. + std::string value; + value.push_back(static_cast((0x07 << 2) | 0x03)); + value.append({static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0x7F)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + ASSERT_NOK(variant->ToJson()); + } +} + +TEST_F(GenericVariantTest, OverLimitInputRejectedBeforeAllocation) { + // The size check must run before the buffers are allocated/copied; the fake-length view is + // never dereferenced. + char byte = 0; + std::string_view huge(&byte, static_cast(VariantDefs::kSizeLimit) + 1); + ASSERT_NOK(GenericVariant::Create(huge, std::string_view(&byte, 1), pool_)); + ASSERT_NOK(GenericVariant::Create(std::string_view(&byte, 1), huge, pool_)); +} + +TEST_F(GenericVariantTest, MalformedInput) { + ASSERT_NOK(GenericVariant::FromJson("", pool_)); + ASSERT_NOK(GenericVariant::FromJson("{", pool_)); + ASSERT_NOK(GenericVariant::FromJson("{\"a\":}", pool_)); + ASSERT_NOK(GenericVariant::FromJson("NaN", pool_)); + + // Unsupported metadata version. + std::string bad_metadata = std::string("\x02\x00\x00", 3); + std::string value = std::string("\x00", 1); + ASSERT_NOK(GenericVariant::Create(value, bad_metadata, pool_)); + // Empty metadata. + ASSERT_NOK(GenericVariant::Create(value, std::string(), pool_)); +} + +TEST_F(GenericVariantTest, SizeInBytesAndViews) { + auto variant = FromJson(R"({"a": 1, "b": "hello"})"); + ASSERT_EQ(variant->SizeInBytes(), + static_cast(variant->RawValue().size() + variant->Metadata().size())); + // A sub-variant shares buffers with its parent and reports a positive position. + ASSERT_OK_AND_ASSIGN(std::shared_ptr b, variant->GetFieldByKey("b")); + ASSERT_GT(b->Pos(), 0); + ASSERT_OK_AND_ASSIGN(std::string_view b_slice, b->Value()); + ASSERT_EQ(ToHex(b_slice), "1568656c6c6f"); +} + +TEST_F(GenericVariantTest, AppendVariantRebuild) { + // Rebuilding a sub-variant through a fresh builder produces a self-contained variant. + auto variant = FromJson(R"({"outer": {"x": [1, 2], "y": "z"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr outer, variant->GetFieldByKey("outer")); + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendVariant(*outer)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rebuilt, builder.Build(pool_)); + ASSERT_EQ(rebuilt->Pos(), 0); + ASSERT_OK_AND_ASSIGN(std::string to_json, rebuilt->ToJson()); + ASSERT_EQ(to_json, "{\"x\":[1,2],\"y\":\"z\"}"); +} + +TEST_F(GenericVariantTest, TimestampAndSpecialTypesToJson) { + // JSON can't produce date/timestamp/binary/uuid variants; build them directly. + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendDate(19737)); // 2024-01-15 + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendTimestamp(1705312496123456LL)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15 09:54:56.123456+00:00\""); + ASSERT_OK_AND_ASSIGN(std::string json_shanghai, v->ToJson("Asia/Shanghai")); + ASSERT_EQ(json_shanghai, "\"2024-01-15 17:54:56.123456+08:00\""); + ASSERT_OK_AND_ASSIGN(std::string json_offset, v->ToJson("+08:00")); + ASSERT_EQ(json_offset, "\"2024-01-15 17:54:56.123456+08:00\""); + ASSERT_NOK(v->ToJson("Not/AZone")); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendTimestampNtz(1705312496000000LL)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15 09:54:56\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendBinary(std::string_view("\x01\x02\x03", 3))); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"AQID\""); + } + { + VariantBuilder builder(false); + std::string uuid_bytes = + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); + ASSERT_OK(builder.AppendUuid(uuid_bytes)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"123e4567-e89b-12d3-a456-426614174000\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendFloat(1.5f)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "1.5"); + } +} + +TEST_F(GenericVariantTest, NonFiniteDoubleToJson) { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendDouble(std::numeric_limits::infinity())); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"Infinity\""); +} + +TEST_F(GenericVariantTest, GetTypeInfoReturnsHeaderBits) { + // GetTypeInfo exposes the primitive header's type-info bits; 42 is encoded as an int1. + auto v = FromJson("42"); + ASSERT_OK_AND_ASSIGN(int32_t type_info, v->GetTypeInfo()); + EXPECT_EQ(type_info, VariantDefs::kInt1); +} + +TEST_F(GenericVariantTest, TypedGettersRejectMismatchedTypes) { + // Each accessor validates the value header and fails when the stored type differs. + auto number = FromJson("42"); // primitive int1 + auto text = FromJson("\"hi\""); // short string + auto real = FromJson("1.5e0"); // double + + // A primitive long is neither boolean/double/decimal/float/binary/string/uuid, nor a + // container. + ASSERT_NOK(number->GetBoolean()); + ASSERT_NOK(number->GetDouble()); + ASSERT_NOK(number->GetDecimal()); + ASSERT_NOK(number->GetFloat()); + ASSERT_NOK(number->GetBinary()); + ASSERT_NOK(number->GetString()); + ASSERT_NOK(number->GetUuid()); + ASSERT_NOK(number->ObjectSize()); + ASSERT_NOK(number->ArraySize()); + // A short string is not a primitive, so long/decimal decoding rejects it early. + ASSERT_NOK(text->GetLong()); + ASSERT_NOK(text->GetDecimal()); + // A double is a primitive but not an integer-like type. + ASSERT_NOK(real->GetLong()); +} + +TEST_F(GenericVariantTest, ValueSizeCoversAllPrimitiveWidths) { + // Builds an array whose elements span the primitive width branches of `ValueSize` (int4, + // decimal8, binary, uuid). Copying elements into the array and re-reading each element's + // value both exercise `ValueSize`. + auto build = [this](const std::function& append) { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + EXPECT_OK(append(builder)); + auto result = builder.Build(pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + }; + std::shared_ptr int4 = + build([](VariantBuilder& b) { return b.AppendLong(100000); }); // needs 4 bytes + std::shared_ptr decimal8 = + build([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{1234567890, 2}); }); + std::shared_ptr binary = + build([](VariantBuilder& b) { return b.AppendBinary(std::string_view("abc", 3)); }); + std::string uuid_bytes(16, '\x07'); + std::shared_ptr uuid = + build([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); }); + + VariantBuilder array_builder(/*allow_duplicate_keys=*/false); + int32_t start = array_builder.GetWritePos(); + std::vector offsets; + for (const std::shared_ptr* element : {&int4, &decimal8, &binary, &uuid}) { + offsets.push_back(array_builder.GetWritePos() - start); + ASSERT_OK(array_builder.AppendVariant(**element)); + } + ASSERT_OK(array_builder.FinishWritingArray(start, offsets)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array_variant, array_builder.Build(pool_)); + + ASSERT_OK_AND_ASSIGN(int32_t size, array_variant->ArraySize()); + ASSERT_EQ(size, 4); + for (int32_t i = 0; i < size; ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr element, + array_variant->GetElementAtIndex(i)); + ASSERT_OK(element->Value()); + } +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp new file mode 100644 index 00000000..c1aee113 --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp @@ -0,0 +1,778 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/common/data/variant/variant_type_utils.h" + +namespace paimon { + +constexpr int32_t kMaxRowFieldSize = 1000; + +// The inference type lattice. A scalar node holds an arrow type (`arrow::null()` is the untyped +// VARIANT sentinel); object nodes track per-field occurrence counts so that rare fields can be +// dropped in the final schema. +struct InferVariantShreddingSchema::SimpleSchema { + struct Field { + std::string name; + std::shared_ptr schema; + double count; + }; + + bool is_object = false; + bool is_array = false; + std::vector fields; + std::shared_ptr element; + std::shared_ptr scalar; + + static std::shared_ptr Variant() { + auto schema = std::make_shared(); + schema->scalar = arrow::null(); + return schema; + } + + static std::shared_ptr Scalar(std::shared_ptr type) { + auto schema = std::make_shared(); + schema->scalar = std::move(type); + return schema; + } +}; + +namespace { + +using SimpleSchema = InferVariantShreddingSchema::SimpleSchema; + +std::shared_ptr MergeSchema(const std::shared_ptr& s1, + const std::shared_ptr& s2); + +// Merges two decimals with possibly different scales. +std::shared_ptr MergeDecimal(const arrow::Decimal128Type& d1, + const arrow::Decimal128Type& d2) { + int32_t scale = std::max(d1.scale(), d2.scale()); + int32_t range = std::max(d1.precision() - d1.scale(), d2.precision() - d2.scale()); + if (range + scale > VariantDefs::kMaxDecimal16Precision) { + // Decimal cannot support precision > 38. + return SimpleSchema::Variant(); + } + return SimpleSchema::Scalar(arrow::decimal128(range + scale, scale)); +} + +std::shared_ptr MergeDecimalWithLong(const arrow::Decimal128Type& d) { + if (d.scale() == 0 && d.precision() <= 18) { + return SimpleSchema::Scalar(arrow::int64()); + } + // A long can always fit in a decimal(19, 0). + auto long_decimal = std::static_pointer_cast(arrow::decimal128(19, 0)); + return MergeDecimal(d, *long_decimal); +} + +std::shared_ptr MergeObjects(const std::shared_ptr& s1, + const std::shared_ptr& s2) { + auto result = std::make_shared(); + result->is_object = true; + size_t f1_idx = 0; + size_t f2_idx = 0; + while (f1_idx < s1->fields.size() && f2_idx < s2->fields.size() && + result->fields.size() < kMaxRowFieldSize) { + const auto& field1 = s1->fields[f1_idx]; + const auto& field2 = s2->fields[f2_idx]; + int32_t comp = field1.name.compare(field2.name); + if (comp == 0) { + result->fields.push_back(SimpleSchema::Field{field1.name, + MergeSchema(field1.schema, field2.schema), + field1.count + field2.count}); + ++f1_idx; + ++f2_idx; + } else if (comp < 0) { + result->fields.push_back(field1); + ++f1_idx; + } else { + result->fields.push_back(field2); + ++f2_idx; + } + } + while (f1_idx < s1->fields.size() && result->fields.size() < kMaxRowFieldSize) { + result->fields.push_back(s1->fields[f1_idx++]); + } + while (f2_idx < s2->fields.size() && result->fields.size() < kMaxRowFieldSize) { + result->fields.push_back(s2->fields[f2_idx++]); + } + return result; +} + +std::shared_ptr MergeSchema(const std::shared_ptr& s1, + const std::shared_ptr& s2) { + // Allow null (missing) to merge into any typed schema. + if (s1 == nullptr) { + return s2; + } + if (s2 == nullptr) { + return s1; + } + if (s1->is_object && s2->is_object) { + return MergeObjects(s1, s2); + } + if (s1->is_array && s2->is_array) { + auto result = std::make_shared(); + result->is_array = true; + result->element = MergeSchema(s1->element, s2->element); + if (result->element == nullptr) { + result->element = SimpleSchema::Variant(); + } + return result; + } + if (s1->scalar != nullptr && s2->scalar != nullptr) { + bool s1_decimal = s1->scalar->id() == arrow::Type::DECIMAL128; + bool s2_decimal = s2->scalar->id() == arrow::Type::DECIMAL128; + bool s1_long = s1->scalar->id() == arrow::Type::INT64; + bool s2_long = s2->scalar->id() == arrow::Type::INT64; + if (s1_decimal && s2_decimal) { + return MergeDecimal(static_cast(*s1->scalar), + static_cast(*s2->scalar)); + } + if (s1_decimal && s2_long) { + return MergeDecimalWithLong(static_cast(*s1->scalar)); + } + if (s1_long && s2_decimal) { + return MergeDecimalWithLong(static_cast(*s2->scalar)); + } + if (s1->scalar->Equals(*s2->scalar)) { + return s1; + } + } + return SimpleSchema::Variant(); +} + +// Returns an appropriate schema for shredding a variant value. Unlike a generic schema-of +// expression, the merged types stay consistent with what shredding allows (e.g. an integer and a +// double merge to VARIANT, not double). +Result> SchemaOf(const GenericVariant& variant, int32_t max_depth) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, variant.GetType()); + switch (type) { + case VariantValueType::kObject: { + if (max_depth <= 0) { + return SimpleSchema::Variant(); + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ObjectSize()); + auto result = std::make_shared(); + result->is_object = true; + result->fields.reserve(size); + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant.GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant("an object field is missing"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_schema, + SchemaOf(*field->value, max_depth - 1)); + result->fields.push_back(SimpleSchema::Field{field->key, field_schema, 1}); + } + // According to the variant spec, object fields must be sorted alphabetically. + for (size_t i = 1; i < result->fields.size(); ++i) { + if (result->fields[i - 1].name.compare(result->fields[i].name) >= 0) { + return Status::Invalid("Variant object fields must be sorted alphabetically"); + } + } + return result; + } + case VariantValueType::kArray: { + if (max_depth <= 0) { + return SimpleSchema::Variant(); + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ArraySize()); + std::shared_ptr element_type; + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant.GetElementAtIndex(i)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_schema, + SchemaOf(*element, max_depth - 1)); + element_type = MergeSchema(element_type, element_schema); + } + auto result = std::make_shared(); + result->is_array = true; + result->element = element_type == nullptr ? SimpleSchema::Variant() : element_type; + return result; + } + case VariantValueType::kNull: + return std::shared_ptr(nullptr); + case VariantValueType::kBoolean: + return SimpleSchema::Scalar(arrow::boolean()); + case VariantValueType::kLong: { + // Compute the smallest decimal that can contain this value. + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + VariantDecimal decimal{value, 0}; + int32_t precision = decimal.Precision(); + if (precision <= 18) { + return SimpleSchema::Scalar(arrow::decimal128(precision, 0)); + } + return SimpleSchema::Scalar(arrow::int64()); + } + case VariantValueType::kString: + return SimpleSchema::Scalar(arrow::utf8()); + case VariantValueType::kDouble: + return SimpleSchema::Scalar(arrow::float64()); + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal decimal, variant.GetDecimal()); + if (decimal.scale < 0) { + // GetDecimal strips trailing zeros and can return a negative scale (100.00 -> + // 1E+2), which neither the shredded parquet type nor reassembly's AppendDecimal + // can represent; scale the value back up. Bounded by construction: the encoded + // decimal had at most 38 digits before the point. + for (; decimal.scale < 0; ++decimal.scale) { + decimal.unscaled *= 10; + } + } + int32_t precision = decimal.Precision(); + int32_t scale = decimal.scale; + // Ensure precision is at least scale (and at least 1) to be valid. + if (precision < scale) { + precision = scale; + } + if (precision == 0) { + precision = 1; + } + return SimpleSchema::Scalar(arrow::decimal128(precision, scale)); + } + case VariantValueType::kDate: + case VariantValueType::kTimestamp: + case VariantValueType::kTimestampNtz: + // The shredding schema builder rejects temporal leaf types (as the Java one does), + // so inferring them would abort the whole write; keep such values in the untyped + // column instead. + return SimpleSchema::Scalar(arrow::null()); + case VariantValueType::kFloat: + return SimpleSchema::Scalar(arrow::float32()); + case VariantValueType::kBinary: + return SimpleSchema::Scalar(arrow::binary()); + default: + return SimpleSchema::Variant(); + } +} + +// Finalizes the inferred schema: 1) widen integer types to int64, 2) replace empty objects with +// VARIANT, 3) limit the total number of shredded fields in the schema. +std::shared_ptr FinalizeSimpleSchema( + const std::shared_ptr& schema, double min_cardinality, + InferVariantShreddingSchema::MaxFields* max_fields) { + // Every field uses a value column. + --max_fields->remaining; + if (max_fields->remaining <= 0) { + return arrow::null(); + } + if (schema == nullptr || + (schema->scalar != nullptr && schema->scalar->id() == arrow::Type::NA)) { + return arrow::null(); + } + if (schema->is_object) { + arrow::FieldVector new_fields; + for (const auto& field : schema->fields) { + if (field.count >= min_cardinality && max_fields->remaining > 0) { + auto new_type = FinalizeSimpleSchema(field.schema, min_cardinality, max_fields); + new_fields.push_back(arrow::field(field.name, new_type)); + } + } + if (!new_fields.empty()) { + return arrow::struct_(new_fields); + } + return arrow::null(); + } + if (schema->is_array) { + auto new_element = FinalizeSimpleSchema(schema->element, min_cardinality, max_fields); + return arrow::list(new_element); + } + switch (schema->scalar->id()) { + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + --max_fields->remaining; + return arrow::int64(); + case arrow::Type::DECIMAL128: { + const auto& decimal_type = static_cast(*schema->scalar); + --max_fields->remaining; + if (decimal_type.precision() <= 18 && decimal_type.scale() == 0) { + return arrow::int64(); + } + if (decimal_type.precision() <= 18) { + return arrow::decimal128(18, decimal_type.scale()); + } + return arrow::decimal128(VariantDefs::kMaxDecimal16Precision, decimal_type.scale()); + } + default: + // All other scalar types use typed_value. + --max_fields->remaining; + return schema->scalar; + } +} + +std::shared_ptr ScaleFieldCounts(const std::shared_ptr& schema, + double scale) { + if (schema == nullptr) { + return nullptr; + } + auto result = std::make_shared(*schema); + if (schema->is_object) { + result->fields.clear(); + result->fields.reserve(schema->fields.size()); + for (const auto& field : schema->fields) { + result->fields.push_back(SimpleSchema::Field{ + field.name, ScaleFieldCounts(field.schema, scale), field.count * scale}); + } + } else if (schema->is_array) { + result->element = ScaleFieldCounts(schema->element, scale); + } + return result; +} + +InferVariantShreddingSchema::ColumnEvidence ScaleToAtMost( + const InferVariantShreddingSchema::ColumnEvidence& evidence, int32_t max_root_value_count) { + if (evidence.root_value_count <= 0 || + evidence.root_value_count <= static_cast(max_root_value_count)) { + return evidence; + } + double scale = static_cast(max_root_value_count) / evidence.root_value_count; + return InferVariantShreddingSchema::ColumnEvidence{ + static_cast(max_root_value_count), + ScaleFieldCounts(evidence.observed_schema, scale)}; +} + +const SimpleSchema::Field* FindSimpleField(const std::shared_ptr& schema, + const std::string& name) { + if (schema == nullptr || !schema->is_object) { + return nullptr; + } + auto it = std::lower_bound(schema->fields.begin(), schema->fields.end(), name, + [](const SimpleSchema::Field& field, const std::string& target) { + return field.name < target; + }); + return it != schema->fields.end() && it->name == name ? &*it : nullptr; +} + +std::shared_ptr FindArrowField(const std::shared_ptr& type, + const std::string& name) { + if (type == nullptr || type->id() != arrow::Type::STRUCT) { + return nullptr; + } + return std::static_pointer_cast(type)->GetFieldByName(name); +} + +bool IsUntyped(const std::shared_ptr& schema) { + return schema == nullptr || + (schema->scalar != nullptr && schema->scalar->id() == arrow::Type::NA); +} + +std::shared_ptr MergeArrowScalars(const std::shared_ptr& first, + const std::shared_ptr& second) { + if (first == nullptr) { + return second; + } + if (second == nullptr) { + return first; + } + bool first_decimal = first->id() == arrow::Type::DECIMAL128; + bool second_decimal = second->id() == arrow::Type::DECIMAL128; + if (first_decimal && second_decimal) { + auto merged = MergeDecimal(static_cast(*first), + static_cast(*second)); + return merged->scalar; + } + if (first_decimal && second->id() == arrow::Type::INT64) { + return MergeDecimalWithLong(static_cast(*first))->scalar; + } + if (first->id() == arrow::Type::INT64 && second_decimal) { + return MergeDecimalWithLong(static_cast(*second))->scalar; + } + return first->Equals(*second) ? first : arrow::null(); +} + +std::shared_ptr WidenScalar(const std::shared_ptr& type) { + if (type == nullptr) { + return arrow::null(); + } + if (type->id() == arrow::Type::DECIMAL128) { + const auto& decimal = static_cast(*type); + if (decimal.precision() <= 18 && decimal.scale() == 0) { + return arrow::int64(); + } + return arrow::decimal128( + decimal.precision() <= 18 ? 18 : VariantDefs::kMaxDecimal16Precision, decimal.scale()); + } + return type; +} + +bool CompatibleTypeFamilies(const std::shared_ptr& previous, + const std::shared_ptr& current) { + if (previous == nullptr || current == nullptr) { + return true; + } + if (previous->id() == arrow::Type::STRUCT || current->is_object) { + return previous->id() == arrow::Type::STRUCT && current->is_object; + } + if (previous->id() == arrow::Type::LIST || current->is_array) { + return previous->id() == arrow::Type::LIST && current->is_array; + } + if (current->scalar == nullptr) { + return false; + } + return MergeArrowScalars(previous, current->scalar)->id() != arrow::Type::NA; +} + +std::shared_ptr SelectedSchemaToSimpleSchema( + const std::shared_ptr& selected, double field_count) { + if (selected == nullptr || selected->id() == arrow::Type::NA) { + return SimpleSchema::Variant(); + } + if (selected->id() == arrow::Type::STRUCT) { + auto result = std::make_shared(); + result->is_object = true; + for (const std::shared_ptr& field : selected->fields()) { + result->fields.push_back(SimpleSchema::Field{ + field->name(), SelectedSchemaToSimpleSchema(field->type(), field_count), + field_count}); + } + return result; + } + if (selected->id() == arrow::Type::LIST) { + auto result = std::make_shared(); + result->is_array = true; + result->element = SelectedSchemaToSimpleSchema( + std::static_pointer_cast(selected)->value_type(), field_count); + return result; + } + return SimpleSchema::Scalar(selected); +} + +std::shared_ptr FinalizeAdaptiveSchema( + std::shared_ptr combined, const std::shared_ptr& current, + std::shared_ptr previous_selected, double root_value_count, + double admission_ratio, double retention_ratio, + InferVariantShreddingSchema::MaxFields* max_fields) { + --max_fields->remaining; + if (max_fields->remaining <= 0) { + return arrow::null(); + } + + if (current != nullptr && previous_selected != nullptr && + !CompatibleTypeFamilies(previous_selected, current)) { + combined = current; + previous_selected = nullptr; + } + if (IsUntyped(combined)) { + if (!IsUntyped(current)) { + combined = current; + } else if (previous_selected != nullptr) { + // Match Java's `combined = previousSelected`: retain the previous selection as the + // input to the remaining recursive width-budget and field-ordering logic. + combined = SelectedSchemaToSimpleSchema(previous_selected, root_value_count); + } else { + return arrow::null(); + } + } + + if (combined->is_object) { + struct Candidate { + const SimpleSchema::Field* field; + double ratio; + bool is_new; + }; + std::vector candidates; + for (const auto& field : combined->fields) { + bool is_new = FindArrowField(previous_selected, field.name) == nullptr; + double threshold = is_new ? admission_ratio : retention_ratio; + double ratio = root_value_count == 0 ? 0 : field.count / root_value_count; + if (ratio >= threshold) { + candidates.push_back(Candidate{&field, ratio, is_new}); + } + } + std::sort(candidates.begin(), candidates.end(), + [](const Candidate& left, const Candidate& right) { + if (left.ratio != right.ratio) { + return left.ratio > right.ratio; + } + if (left.is_new != right.is_new) { + return !left.is_new; + } + return left.field->name < right.field->name; + }); + + arrow::FieldVector selected; + for (const Candidate& candidate : candidates) { + if (max_fields->remaining <= 0) { + break; + } + const SimpleSchema::Field* current_field = + FindSimpleField(current, candidate.field->name); + std::shared_ptr previous_field = + FindArrowField(previous_selected, candidate.field->name); + std::shared_ptr selected_type = FinalizeAdaptiveSchema( + candidate.field->schema, current_field == nullptr ? nullptr : current_field->schema, + previous_field == nullptr ? nullptr : previous_field->type(), root_value_count, + admission_ratio, retention_ratio, max_fields); + selected.push_back(arrow::field(candidate.field->name, selected_type)); + } + std::sort(selected.begin(), selected.end(), + [](const std::shared_ptr& left, + const std::shared_ptr& right) { + return left->name() < right->name(); + }); + return selected.empty() ? arrow::null() : arrow::struct_(selected); + } + + if (combined->is_array) { + std::shared_ptr current_element = + current != nullptr && current->is_array ? current->element : nullptr; + std::shared_ptr previous_element; + if (previous_selected != nullptr && previous_selected->id() == arrow::Type::LIST) { + previous_element = + std::static_pointer_cast(previous_selected)->value_type(); + } + return arrow::list(FinalizeAdaptiveSchema(combined->element, current_element, + previous_element, root_value_count, + admission_ratio, retention_ratio, max_fields)); + } + + --max_fields->remaining; + std::shared_ptr current_scalar = + current == nullptr ? nullptr : current->scalar; + if (current_scalar == nullptr) { + return previous_selected == nullptr ? WidenScalar(combined->scalar) : previous_selected; + } + if (previous_selected == nullptr) { + return WidenScalar(current_scalar); + } + std::shared_ptr merged = MergeArrowScalars(previous_selected, current_scalar); + return merged->id() == arrow::Type::NA ? WidenScalar(current_scalar) : merged; +} + +} // namespace + +InferVariantShreddingSchema::InferVariantShreddingSchema( + const std::shared_ptr& logical_schema, const std::shared_ptr& pool, + int32_t max_schema_width, int32_t max_schema_depth, double min_field_cardinality_ratio) + : logical_schema_(logical_schema), + pool_(pool), + max_schema_width_(max_schema_width), + max_schema_depth_(max_schema_depth), + min_field_cardinality_ratio_(min_field_cardinality_ratio) { + Path current; + CollectVariantPaths(logical_schema_->fields(), ¤t, &paths_to_variant_); +} + +void InferVariantShreddingSchema::CollectVariantPaths( + const std::vector>& fields, Path* current, + std::vector* paths) { + for (int32_t i = 0; i < static_cast(fields.size()); ++i) { + const std::shared_ptr& field = fields[i]; + current->push_back(i); + if (VariantTypeUtils::IsVariantField(field)) { + paths->push_back(*current); + } else if (field->type()->id() == arrow::Type::STRUCT) { + CollectVariantPaths(field->type()->fields(), current, paths); + } + current->pop_back(); + } +} + +Result>> +InferVariantShreddingSchema::CollectSamplesAtPath(const SampleBatches& sample_batches, + const Path& path) const { + std::vector> samples; + for (const std::shared_ptr& sample_batch : sample_batches) { + std::shared_ptr column = sample_batch; + // Child slots below a null struct row are unspecified in Arrow and must not be decoded. + std::vector> ancestors; + for (int32_t index : path) { + if (column == nullptr || column->type_id() != arrow::Type::STRUCT) { + return Status::Invalid("sample batch does not match the variant column path"); + } + if (column != sample_batch) { + ancestors.push_back(column); + } + column = arrow::internal::checked_cast(*column).field(index); + } + if (column == nullptr) { + return Status::Invalid("sample batch misses the planned variant column"); + } + const auto& variant_array = + arrow::internal::checked_cast(*column); + const auto& value_array = + arrow::internal::checked_cast(*variant_array.field(0)); + const auto& metadata_array = + arrow::internal::checked_cast(*variant_array.field(1)); + for (int64_t row = 0; row < variant_array.length(); ++row) { + bool row_is_null = variant_array.IsNull(row); + for (const std::shared_ptr& ancestor : ancestors) { + row_is_null = row_is_null || ancestor->IsNull(row); + } + if (row_is_null) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(std::string_view(value_array.GetView(row)), + std::string_view(metadata_array.GetView(row)), pool_)); + samples.push_back(std::move(variant)); + } + } + return samples; +} + +Result> InferVariantShreddingSchema::CreatePhysicalSchema( + const SelectedSchemas& selected_schemas) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + VariantShreddingWritePlan::CreateFromPaths(logical_schema_, selected_schemas)); + return plan->PhysicalSchema(); +} + +Result> InferVariantShreddingSchema::InferSchema( + const SampleBatches& samples) const { + PAIMON_ASSIGN_OR_RAISE(AdaptiveInferenceResult result, + InferInitial(samples, std::numeric_limits::max())); + return result.physical_schema; +} + +Result +InferVariantShreddingSchema::InferInitial(const SampleBatches& samples, + int32_t effective_sample_size) const { + MaxFields max_fields = CreateMaxFieldsBudget(); + InferenceEvidence evidence; + SelectedSchemas selected_schemas; + for (const Path& path : paths_to_variant_) { + PAIMON_ASSIGN_OR_RAISE(std::vector> column_samples, + CollectSamplesAtPath(samples, path)); + PAIMON_ASSIGN_OR_RAISE( + AdaptiveColumnResult result, + InferInitialColumn(column_samples, effective_sample_size, &max_fields)); + evidence.columns.emplace(path, std::move(result.evidence)); + selected_schemas.emplace(path, std::move(result.selected_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + CreatePhysicalSchema(selected_schemas)); + return AdaptiveInferenceResult{std::move(physical_schema), std::move(evidence), + std::move(selected_schemas)}; +} + +Result +InferVariantShreddingSchema::InferAdaptive(const InferenceEvidence& previous_evidence, + const SelectedSchemas& previous_selected_schemas, + const SampleBatches& samples, + int32_t effective_sample_size, double admission_ratio, + double retention_ratio) const { + MaxFields max_fields = CreateMaxFieldsBudget(); + InferenceEvidence evidence; + SelectedSchemas selected_schemas; + for (const Path& path : paths_to_variant_) { + PAIMON_ASSIGN_OR_RAISE(std::vector> column_samples, + CollectSamplesAtPath(samples, path)); + auto evidence_it = previous_evidence.columns.find(path); + ColumnEvidence previous_column; + if (evidence_it != previous_evidence.columns.end()) { + previous_column = evidence_it->second; + } + auto selected_it = previous_selected_schemas.find(path); + std::shared_ptr previous_selected = + selected_it == previous_selected_schemas.end() ? nullptr : selected_it->second; + PAIMON_ASSIGN_OR_RAISE(AdaptiveColumnResult result, + InferAdaptiveColumn(previous_column, previous_selected, + column_samples, effective_sample_size, + admission_ratio, retention_ratio, &max_fields)); + evidence.columns.emplace(path, std::move(result.evidence)); + selected_schemas.emplace(path, std::move(result.selected_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + CreatePhysicalSchema(selected_schemas)); + return AdaptiveInferenceResult{std::move(physical_schema), std::move(evidence), + std::move(selected_schemas)}; +} + +Result> InferVariantShreddingSchema::InferColumnShreddingType( + const std::vector>& samples, MaxFields* max_fields) const { + PAIMON_ASSIGN_OR_RAISE(ColumnEvidence evidence, AnalyzeColumn(samples)); + double min_cardinality = std::ceil(evidence.root_value_count * min_field_cardinality_ratio_); + std::shared_ptr finalized = + FinalizeSimpleSchema(evidence.observed_schema, min_cardinality, max_fields); + if (finalized->id() == arrow::Type::NA) { + return std::shared_ptr(nullptr); + } + return finalized; +} + +Result InferVariantShreddingSchema::AnalyzeColumn( + const std::vector>& samples) const { + ColumnEvidence evidence; + for (const auto& sample : samples) { + if (sample == nullptr) { + continue; + } + ++evidence.root_value_count; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_schema, + SchemaOf(*sample, max_schema_depth_)); + evidence.observed_schema = MergeSchema(evidence.observed_schema, row_schema); + } + return evidence; +} + +Result +InferVariantShreddingSchema::InferInitialColumn( + const std::vector>& samples, int32_t effective_sample_size, + MaxFields* max_fields) const { + PAIMON_ASSIGN_OR_RAISE(ColumnEvidence evidence, AnalyzeColumn(samples)); + double min_cardinality = std::ceil(evidence.root_value_count * min_field_cardinality_ratio_); + std::shared_ptr finalized = + FinalizeSimpleSchema(evidence.observed_schema, min_cardinality, max_fields); + return AdaptiveColumnResult{ScaleToAtMost(evidence, effective_sample_size), finalized}; +} + +Result +InferVariantShreddingSchema::InferAdaptiveColumn( + const ColumnEvidence& previous_evidence, + const std::shared_ptr& previous_selected, + const std::vector>& samples, int32_t effective_sample_size, + double admission_ratio, double retention_ratio, MaxFields* max_fields) const { + PAIMON_ASSIGN_OR_RAISE(ColumnEvidence current, AnalyzeColumn(samples)); + ColumnEvidence combined; + if (current.root_value_count == 0) { + combined = previous_evidence; + } else { + ColumnEvidence bounded_previous = ScaleToAtMost(previous_evidence, effective_sample_size); + combined.root_value_count = current.root_value_count + bounded_previous.root_value_count; + combined.observed_schema = + MergeSchema(bounded_previous.observed_schema, current.observed_schema); + combined = ScaleToAtMost(combined, effective_sample_size); + } + std::shared_ptr selected = FinalizeAdaptiveSchema( + combined.observed_schema, current.observed_schema, previous_selected, + combined.root_value_count, admission_ratio, retention_ratio, max_fields); + return AdaptiveColumnResult{std::move(combined), std::move(selected)}; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.h b/src/paimon/common/data/variant/infer_variant_shredding_schema.h new file mode 100644 index 00000000..8d8a45bc --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.h @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class DataType; +class Field; +class Schema; +} // namespace arrow + +namespace paimon { + +class MemoryPool; +class VariantShreddingInferenceSession; + +/// Infers the complete physical write schema from sampled logical rows (mirroring the Java +/// `InferVariantShreddingSchema`). Rare fields (below the cardinality ratio) stay in the +/// un-shredded variant binary, integer types widen to int64, and the total number of shredded +/// fields is limited across all Variant columns. +class InferVariantShreddingSchema { + public: + using Path = std::vector; + using SampleBatches = std::vector>; + + struct SimpleSchema; + + struct ColumnEvidence { + double root_value_count = 0; + std::shared_ptr observed_schema; + }; + + struct AdaptiveColumnResult { + ColumnEvidence evidence; + /// arrow::null() means the column remains unshredded. + std::shared_ptr selected_schema; + }; + + /// The mutable budget of shredded fields remaining. One instance is shared across all + /// variant columns of a schema so that the total inferred width stays within + /// `variant.shredding.maxSchemaWidth` (mirroring the Java `MaxFields`). + struct MaxFields { + int32_t remaining; + }; + + InferVariantShreddingSchema(const std::shared_ptr& logical_schema, + const std::shared_ptr& pool, int32_t max_schema_width, + int32_t max_schema_depth, double min_field_cardinality_ratio); + + /// Infers one complete physical schema from the sampled logical row batches. + Result> InferSchema(const SampleBatches& samples) const; + + /// Creates the shared shredded-field budget for one schema inference. + MaxFields CreateMaxFieldsBudget() const { + return MaxFields{max_schema_width_}; + } + + /// Infers the shredding type of one variant column from its sampled non-null values, e.g. + /// `struct{a: int64, b: string}`. `arrow::null()` leaves denote untyped variant sub-values. + /// `max_fields` is the budget shared across all columns of the schema. Returns nullptr when + /// no useful shredding schema was found (the column should stay unshredded). + Result> InferColumnShreddingType( + const std::vector>& samples, MaxFields* max_fields) const; + + /// Initial/adaptive inference primitives used by a rolling-writer-scoped session. + Result InferInitialColumn( + const std::vector>& samples, int32_t effective_sample_size, + MaxFields* max_fields) const; + + Result InferAdaptiveColumn( + const ColumnEvidence& previous_evidence, + const std::shared_ptr& previous_selected, + const std::vector>& samples, int32_t effective_sample_size, + double admission_ratio, double retention_ratio, MaxFields* max_fields) const; + + private: + friend class VariantShreddingInferenceSession; + + struct InferenceEvidence { + std::map columns; + }; + + using SelectedSchemas = std::map>; + + struct AdaptiveInferenceResult { + std::shared_ptr physical_schema; + InferenceEvidence evidence; + SelectedSchemas selected_schemas; + }; + + Result InferInitial(const SampleBatches& samples, + int32_t effective_sample_size) const; + + Result InferAdaptive(const InferenceEvidence& previous_evidence, + const SelectedSchemas& previous_selected_schemas, + const SampleBatches& samples, + int32_t effective_sample_size, + double admission_ratio, + double retention_ratio) const; + + static void CollectVariantPaths(const std::vector>& fields, + Path* current, std::vector* paths); + + Result>> CollectSamplesAtPath( + const SampleBatches& sample_batches, const Path& path) const; + + Result> CreatePhysicalSchema( + const SelectedSchemas& selected_schemas) const; + + Result AnalyzeColumn( + const std::vector>& samples) const; + + std::shared_ptr logical_schema_; + std::shared_ptr pool_; + std::vector paths_to_variant_; + int32_t max_schema_width_; + int32_t max_schema_depth_; + double min_field_cardinality_ratio_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp new file mode 100644 index 00000000..13e77dbd --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class InferVariantShreddingSchemaTest : public ::testing::Test { + public: + // Infers one column with a fresh shared-width budget. + static Result> InferColumn( + const InferVariantShreddingSchema& infer, + const std::vector>& samples) { + InferVariantShreddingSchema::MaxFields max_fields = infer.CreateMaxFieldsBudget(); + return infer.InferColumnShreddingType(samples, &max_fields); + } + + std::vector> Samples(const std::vector& jsons) { + std::vector> samples; + for (const char* json : jsons) { + if (json == nullptr) { + samples.push_back(nullptr); + continue; + } + auto variant = GenericVariant::FromJson(json, pool_); + EXPECT_TRUE(variant.ok()) << variant.status().ToString(); + samples.push_back(variant.value()); + } + return samples; + } + + // Builds a single variant using the direct append API, which can encode types (float, binary, + // uuid, decimals with a specific scale) that JSON parsing never produces. + std::shared_ptr BuildVariant( + const std::function& append) { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + Status st = append(builder); + EXPECT_TRUE(st.ok()) << st.ToString(); + auto result = builder.Build(pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr empty_logical_schema_ = arrow::schema(arrow::FieldVector{}); + InferVariantShreddingSchema infer_{empty_logical_schema_, pool_, + /*max_schema_width=*/300, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; +}; + +TEST_F(InferVariantShreddingSchemaTest, InferObjectSchema) { + auto samples = Samples({ + R"({"age": 35, "city": "Hangzhou"})", + R"({"age": 20, "city": "Beijing", "tags": [1, 2]})", + nullptr, + R"({"age": 120000000000, "city": "Shanghai"})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // Integers widen to int64, strings stay, arrays of small ints infer as list. + auto expected = + arrow::struct_({arrow::field("age", arrow::int64()), arrow::field("city", arrow::utf8()), + arrow::field("tags", arrow::list(arrow::int64()))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + ASSERT_OK(VariantShreddingUtils::VariantShreddingSchema(inferred)); +} + +TEST_F(InferVariantShreddingSchemaTest, NullObjectFieldMergesWithTypedValue) { + auto samples = Samples({ + R"({"a": 1, "b": null})", + R"({"a": 2, "b": 3})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + auto expected = + arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, MixedTypesFallToVariant) { + auto samples = Samples({ + R"({"x": 1, "y": 1.5e0})", + R"({"x": "string now", "y": 2.5e0})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // x saw both int and string: untyped variant leaf; y stays double (exponent notation + // parses as double, plain decimals parse as DECIMAL). + auto expected = + arrow::struct_({arrow::field("x", arrow::null()), arrow::field("y", arrow::float64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + ASSERT_OK(VariantShreddingUtils::VariantShreddingSchema(inferred)); +} + +TEST_F(InferVariantShreddingSchemaTest, RareFieldsDropped) { + std::vector jsons; + for (int i = 0; i < 19; ++i) { + jsons.push_back("{\"common\": 1}"); + } + jsons.push_back(R"({"common": 2, "rare": true})"); + auto samples = Samples(jsons); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // "rare" appears in 1/20 rows (< 0.1 ratio): dropped from the typed schema. + auto expected = arrow::struct_({arrow::field("common", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, FieldCardinalityAdmissionThreshold) { + std::vector jsons = { + R"({"common": 1, "rare": 99})", + R"({"common": 2, "rare": 88})", + R"({"common": 3})", + R"({"common": 4})", + R"({"common": 5})", + R"({"common": 6})", + R"({"common": 7})", + R"({"common": 8})", + R"({"common": 9})", + R"({"common": 10})", + }; + auto samples = Samples(jsons); + + InferVariantShreddingSchema permissive{empty_logical_schema_, pool_, + /*max_schema_width=*/300, + /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr admitted, + InferColumn(permissive, samples)); + auto expected_admitted = arrow::struct_( + {arrow::field("common", arrow::int64()), arrow::field("rare", arrow::int64())}); + ASSERT_TRUE(admitted->Equals(*expected_admitted)) << admitted->ToString(); + + InferVariantShreddingSchema strict{empty_logical_schema_, pool_, + /*max_schema_width=*/300, + /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.25}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr rejected, InferColumn(strict, samples)); + auto expected_rejected = arrow::struct_({arrow::field("common", arrow::int64())}); + ASSERT_TRUE(rejected->Equals(*expected_rejected)) << rejected->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, DecimalMerging) { + auto samples = Samples({ + "{\"d\": 100.99}", + "{\"d\": 1.5}", + "{\"d\": 42}", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // Decimals merge to a widened decimal (scale 2, enough integer digits), capped at 18 digits. + auto expected = arrow::struct_({arrow::field("d", arrow::decimal128(18, 2))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, AllPrimitiveTypes) { + auto samples = Samples({R"({ + "string": "test", + "long": 123456789, + "double": 3.14159, + "boolean": true, + "null": null + })"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + auto expected = arrow::struct_({ + arrow::field("boolean", arrow::boolean()), + arrow::field("double", arrow::decimal128(18, 5)), + arrow::field("long", arrow::int64()), + arrow::field("null", arrow::null()), + arrow::field("string", arrow::utf8()), + }); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, MixedArrayElementTypesFallToVariant) { + auto samples = Samples({ + R"({"arr": [1, 2, 3]})", + R"({"arr": ["a", "b", "c"]})", + R"({"arr": [true, false, true]})", + R"({"arr": [1, "mixed", true]})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + auto expected = arrow::struct_({arrow::field("arr", arrow::list(arrow::null()))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, NullInNestedArrays) { + auto samples = Samples({ + R"({"arr": [1, 2, 3, null, 5]})", + R"({"arr": [null, null, null]})", + R"({"arr": [10, null, 20, null, 30]})", + R"({"arr": null})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + auto expected = arrow::struct_({arrow::field("arr", arrow::list(arrow::null()))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, LargeDatasetWithManyFields) { + std::vector documents; + documents.reserve(500); + for (int32_t row = 0; row < 500; ++row) { + std::string json = "{"; + for (int32_t field = 0; field < 50; ++field) { + if (field > 0) { + json += ","; + } + json += "\"field" + std::to_string(field) + + "\":" + std::to_string((row * 50 + field) % 1000); + } + json += "}"; + documents.push_back(std::move(json)); + } + std::vector jsons; + jsons.reserve(documents.size()); + for (const std::string& document : documents) { + jsons.push_back(document.c_str()); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(infer_, Samples(jsons))); + ASSERT_EQ(arrow::Type::STRUCT, inferred->id()); + const auto& struct_type = static_cast(*inferred); + ASSERT_EQ(50, struct_type.num_fields()); + for (int32_t field = 0; field < 50; ++field) { + auto inferred_field = struct_type.GetFieldByName("field" + std::to_string(field)); + ASSERT_NE(nullptr, inferred_field); + ASSERT_TRUE(inferred_field->type()->Equals(*arrow::int64())); + } +} + +TEST_F(InferVariantShreddingSchemaTest, NoUsefulSchema) { + auto scalar_samples = Samples({"1", "2"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scalar_inferred, + InferColumn(infer_, scalar_samples)); + ASSERT_NE(scalar_inferred, nullptr); + ASSERT_TRUE(scalar_inferred->Equals(*arrow::int64())) << scalar_inferred->ToString(); + + // Conflicting top-level types stay unshredded. + auto mixed_samples = Samples({"1", "\"a string\""}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr mixed_inferred, + InferColumn(infer_, mixed_samples)); + ASSERT_EQ(mixed_inferred, nullptr); + + // All-null columns stay unshredded. + auto null_samples = Samples({nullptr, nullptr}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr null_inferred, + InferColumn(infer_, null_samples)); + ASSERT_EQ(null_inferred, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, MaxSchemaWidthLimit) { + InferVariantShreddingSchema narrow_infer{empty_logical_schema_, pool_, + /*max_schema_width=*/3, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; + auto samples = Samples({R"({"a": 1, "b": 2, "c": 3, "d": 4, "e": 5})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(narrow_infer, samples)); + ASSERT_NE(inferred, nullptr); + // Budget of 3: the root object costs 1, "a" costs 2 (value + typed_value); the remaining + // fields exceed the budget and are dropped from the typed schema. + auto expected = arrow::struct_({arrow::field("a", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + + // One budget serves all variant columns of a schema: after the first column consumes it, a + // second column cannot shred anymore. + InferVariantShreddingSchema::MaxFields max_fields = narrow_infer.CreateMaxFieldsBudget(); + auto first_samples = Samples({R"({"a": 1, "b": 2})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + narrow_infer.InferColumnShreddingType(first_samples, &max_fields)); + ASSERT_NE(first, nullptr); + ASSERT_TRUE(first->Equals(*expected)) << first->ToString(); + auto second_samples = Samples({R"({"c": 1})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + narrow_infer.InferColumnShreddingType(second_samples, &max_fields)); + ASSERT_EQ(second, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, MaxSchemaDepthLimit) { + InferVariantShreddingSchema shallow_infer{empty_logical_schema_, pool_, + /*max_schema_width=*/300, /*max_schema_depth=*/1, + /*min_field_cardinality_ratio=*/0.1}; + auto samples = Samples({R"({"outer": {"inner": 1}})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(shallow_infer, samples)); + ASSERT_NE(inferred, nullptr); + // Depth 1: the nested object stays an untyped variant leaf. + auto expected = arrow::struct_({arrow::field("outer", arrow::null())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, DeepNestedObjectSchema) { + auto samples = Samples({R"({"level1":{"level2":{"level3":{"value":42}}}})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + auto expected = arrow::struct_({arrow::field( + "level1", + arrow::struct_({arrow::field( + "level2", + arrow::struct_({arrow::field( + "level3", arrow::struct_({arrow::field("value", arrow::int64())}))}))}))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, AdaptivePreviousSelectionHonorsSharedWidthBudget) { + InferVariantShreddingSchema narrow_infer{empty_logical_schema_, pool_, + /*max_schema_width=*/6, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; + // A scalar-to-object transition can leave the combined evidence untyped while selecting the + // current object schema for writing. + InferVariantShreddingSchema::ColumnEvidence previous_evidence; + previous_evidence.root_value_count = 1; + auto previous_selected = + arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())}); + InferVariantShreddingSchema::MaxFields max_fields = narrow_infer.CreateMaxFieldsBudget(); + // In the preceding file, an earlier Variant column used one slot, leaving five slots for this + // column's root, a and b. In this file that earlier column expanded to use three slots, so only + // three remain and the previous selection must be trimmed. + max_fields.remaining = 3; + + ASSERT_OK_AND_ASSIGN( + InferVariantShreddingSchema::AdaptiveColumnResult result, + narrow_infer.InferAdaptiveColumn( + previous_evidence, previous_selected, /*samples=*/{}, /*effective_sample_size=*/10, + /*admission_ratio=*/0.1, /*retention_ratio=*/0.05, &max_fields)); + auto expected = arrow::struct_({arrow::field("a", arrow::int64())}); + ASSERT_TRUE(result.selected_schema->Equals(*expected)) << result.selected_schema->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, TrailingZeroDecimalNormalized) { + // GetDecimal strips 100.00 to 1E+2 (scale -2); the inferred type must carry a non-negative + // scale or reassembling the shredded file would be rejected. After normalization the value + // is an integral decimal, which finalization widens to int64. + auto samples = Samples({"100.00"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + ASSERT_TRUE(inferred->Equals(*arrow::int64())) << inferred->ToString(); + + auto mixed = Samples({"100.00", "1.5"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr mixed_inferred, + InferColumn(infer_, mixed)); + ASSERT_NE(mixed_inferred, nullptr); + ASSERT_TRUE(mixed_inferred->Equals(*arrow::decimal128(18, 1))) << mixed_inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, TemporalValuesStayUnshredded) { + // The shredding schema builder rejects temporal leaf types, so inferring them would abort + // the write; date/timestamp samples must leave the column unshredded. + VariantBuilder date_builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(date_builder.AppendDate(19000)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr date_variant, date_builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr date_inferred, + InferColumn(infer_, {date_variant})); + ASSERT_EQ(date_inferred, nullptr); + + VariantBuilder ts_builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(ts_builder.AppendTimestamp(1700000000000000)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ts_variant, ts_builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ts_inferred, + InferColumn(infer_, {ts_variant})); + ASSERT_EQ(ts_inferred, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, MergeObjectsWithDisjointFields) { + // Objects whose keys interleave exercise both single-side merge branches (field only in the + // first object, and field only in the second). + auto samples = Samples({R"({"a": 1, "c": 3})", R"({"b": 2, "c": 4})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + auto expected = + arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64()), + arrow::field("c", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, VariantNullSamplesAndFields) { + // A top-level variant null (JSON `null`, not a missing sample) merges away, leaving the other + // sample's inferred type. + auto scalar_and_null = Samples({"1", "null"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr merged, + InferColumn(infer_, scalar_and_null)); + ASSERT_NE(merged, nullptr); + ASSERT_TRUE(merged->Equals(*arrow::int64())) << merged->ToString(); + + // An object field that is always variant-null becomes an untyped variant leaf. + auto object_with_null = Samples({R"({"a": null, "b": 1})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(infer_, object_with_null)); + ASSERT_NE(inferred, nullptr); + auto expected = + arrow::struct_({arrow::field("a", arrow::null()), arrow::field("b", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, ArraysMerge) { + // Two arrays merge element-wise into a single typed element schema. + auto samples = Samples({"[1, 2]", "[3, 4]"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + ASSERT_TRUE(inferred->Equals(*arrow::list(arrow::int64()))) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, ArrayBeyondDepthLimitStaysVariant) { + InferVariantShreddingSchema shallow_infer{empty_logical_schema_, pool_, + /*max_schema_width=*/300, /*max_schema_depth=*/1, + /*min_field_cardinality_ratio=*/0.1}; + auto samples = Samples({R"({"arr": [1, 2]})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(shallow_infer, samples)); + ASSERT_NE(inferred, nullptr); + // Depth 1: the nested array is beyond the limit and stays an untyped variant leaf. + auto expected = arrow::struct_({arrow::field("arr", arrow::null())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, ScalarLeafTypes) { + // Float and binary leaves shred to their arrow types. + std::shared_ptr float_variant = + BuildVariant([](VariantBuilder& b) { return b.AppendFloat(1.5f); }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr float_inferred, + InferColumn(infer_, {float_variant})); + ASSERT_NE(float_inferred, nullptr); + ASSERT_TRUE(float_inferred->Equals(*arrow::float32())) << float_inferred->ToString(); + + std::shared_ptr binary_variant = BuildVariant( + [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02", 2)); }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr binary_inferred, + InferColumn(infer_, {binary_variant})); + ASSERT_NE(binary_inferred, nullptr); + ASSERT_TRUE(binary_inferred->Equals(*arrow::binary())) << binary_inferred->ToString(); + + // A UUID has no shredding type, so the column stays unshredded. + std::string uuid_bytes(16, '\0'); + std::shared_ptr uuid_variant = + BuildVariant([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr uuid_inferred, + InferColumn(infer_, {uuid_variant})); + ASSERT_EQ(uuid_inferred, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, LargeIntegerAndDecimalMerging) { + // A 19-digit long exceeds decimal(18) precision, so it stays a genuine int64 leaf. + auto big_long = Samples({"1000000000000000000"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr long_inferred, + InferColumn(infer_, big_long)); + ASSERT_NE(long_inferred, nullptr); + ASSERT_TRUE(long_inferred->Equals(*arrow::int64())) << long_inferred->ToString(); + + // A long (int64) merged with a fractional decimal widens via MergeDecimalWithLong. + auto long_then_decimal = Samples({"1000000000000000000", "1.5"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ld, + InferColumn(infer_, long_then_decimal)); + ASSERT_NE(ld, nullptr); + ASSERT_TRUE(ld->Equals(*arrow::decimal128(38, 1))) << ld->ToString(); + + // The reversed order (decimal first, then long) hits the mirrored branch. + auto decimal_then_long = Samples({"1.5", "1000000000000000000"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr dl, + InferColumn(infer_, decimal_then_long)); + ASSERT_NE(dl, nullptr); + ASSERT_TRUE(dl->Equals(*arrow::decimal128(38, 1))) << dl->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, LongMergedWithIntegralDecimalStaysLong) { + // A scale-0 decimal that fits in 18 digits merges with a long back to int64. + std::shared_ptr integral_decimal = + BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{123, 0}); }); + std::shared_ptr big_long = Samples({"1000000000000000000"})[0]; + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(infer_, {integral_decimal, big_long})); + ASSERT_NE(inferred, nullptr); + ASSERT_TRUE(inferred->Equals(*arrow::int64())) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, SmallFractionalDecimalPrecisionAdjusted) { + // 0.0015 has more fractional digits than significant digits; precision widens up to the scale. + auto samples = Samples({"0.0015"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + ASSERT_TRUE(inferred->Equals(*arrow::decimal128(18, 4))) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, DecimalMergeOverflowFallsToVariant) { + // A 38-digit integral decimal merged with a high-scale decimal would need precision > 38, + // which decimal cannot represent, so the column stays unshredded. + __int128 wide_unscaled = 0; + for (int i = 0; i < 38; ++i) { + wide_unscaled = wide_unscaled * 10 + 1; // 38 ones, no trailing zeros + } + std::shared_ptr wide_decimal = BuildVariant( + [&](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{wide_unscaled, 0}); }); + std::shared_ptr high_scale_decimal = + BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{15, 20}); }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(infer_, {wide_decimal, high_scale_decimal})); + ASSERT_EQ(inferred, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, ObjectWithAllRareFieldsStaysUnshredded) { + InferVariantShreddingSchema strict_infer{empty_logical_schema_, pool_, + /*max_schema_width=*/300, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.6}; + // Two objects with disjoint single-occurrence keys: with a 0.6 ratio every field is below the + // cardinality threshold, so the object contributes no typed field and the column is dropped. + auto samples = Samples({R"({"a": 1})", R"({"b": 2})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(strict_infer, samples)); + ASSERT_EQ(inferred, nullptr); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant.cpp b/src/paimon/common/data/variant/variant.cpp new file mode 100644 index 00000000..455fa266 --- /dev/null +++ b/src/paimon/common/data/variant/variant.cpp @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/variant.h" + +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_get.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +class Variant::Impl { + public: + Impl(std::shared_ptr variant, std::shared_ptr pool) + : variant_(std::move(variant)), pool_(std::move(pool)), arrow_pool_(GetArrowPool(pool_)) {} + + const std::shared_ptr& GetVariant() const { + return variant_; + } + + const std::shared_ptr& GetPool() const { + return pool_; + } + + const std::shared_ptr& GetArrowMemoryPool() const { + return arrow_pool_; + } + + private: + std::shared_ptr variant_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +Variant::Variant(std::unique_ptr&& impl) : impl_(std::move(impl)) {} +Variant::~Variant() = default; + +Result> Variant::FromJson(const std::string& json, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::FromJson(json, pool)); + auto impl = std::make_unique(std::move(variant), pool); + return std::unique_ptr(new Variant(std::move(impl))); +} + +Result> Variant::Create(const char* value, uint64_t value_length, + const char* metadata, uint64_t metadata_length, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(std::string_view(value, value_length), + std::string_view(metadata, metadata_length), pool)); + auto impl = std::make_unique(std::move(variant), pool); + return std::unique_ptr(new Variant(std::move(impl))); +} + +std::string_view Variant::Value() const { + return impl_->GetVariant()->RawValue(); +} + +std::string_view Variant::Metadata() const { + return impl_->GetVariant()->Metadata(); +} + +int64_t Variant::SizeInBytes() const { + return impl_->GetVariant()->SizeInBytes(); +} + +Result Variant::ToJson(const std::string& zone_id) const { + return impl_->GetVariant()->ToJson(zone_id); +} + +Result> Variant::VariantGet(const std::string& path, + struct ArrowSchema* target_type, + const VariantCastArgs& cast_args) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_field, + arrow::ImportField(target_type)); + return VariantGetExecutor::Get(impl_->GetVariant(), path, target_field->type(), cast_args); +} + +Result> Variant::VariantGetArrow( + const std::string& path, struct ArrowSchema* target_field, + const VariantCastArgs& cast_args) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr field, + arrow::ImportField(target_field)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr array, + VariantGetExecutor::GetAsArrow(impl_->GetVariant(), path, field, cast_args, + impl_->GetPool(), impl_->GetArrowMemoryPool())); + auto result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, result.get())); + return result; +} + +Result> Variant::VariantGetJson(const std::string& path, + const std::string& zone_id) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, + VariantGetExecutor::ExtractByPath(impl_->GetVariant(), path)); + if (extracted == nullptr) { + return std::optional(std::nullopt); + } + PAIMON_ASSIGN_OR_RAISE(std::string json, extracted->ToJson(zone_id)); + return std::optional(std::move(json)); +} + +Result> Variant::ArrowField( + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + auto variant_field = VariantTypeUtils::ToArrowField(field_name, nullable, std::move(metadata)); + auto field = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*variant_field, field.get())); + return field; +} + +class VariantAccessBuilder::Impl { + public: + arrow::FieldVector fields; +}; + +VariantAccessBuilder::VariantAccessBuilder() : impl_(std::make_unique()) {} +VariantAccessBuilder::~VariantAccessBuilder() = default; + +Status VariantAccessBuilder::AddField(struct ArrowSchema* target_type, const std::string& path, + bool fail_on_error, const std::string& zone_id) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target, + arrow::ImportField(target_type)); + // Validate the path eagerly so mistakes fail at build time, not at read time. + PAIMON_RETURN_NOT_OK(VariantPathSegment::Parse(path)); + // Keep the target field's own metadata (e.g. the variant extension marker of a + // `Variant::ArrowField` target, which drives the deep re-encode cast) and add the access + // description to it. + std::vector keys = {DataField::DESCRIPTION}; + std::vector values = { + VariantAccessUtils::BuildVariantMetadata(path, fail_on_error, zone_id)}; + if (target->metadata() != nullptr) { + for (int64_t i = 0; i < target->metadata()->size(); ++i) { + if (target->metadata()->key(i) == DataField::DESCRIPTION) { + continue; + } + keys.push_back(target->metadata()->key(i)); + values.push_back(target->metadata()->value(i)); + } + } + impl_->fields.push_back(arrow::field(std::to_string(impl_->fields.size()), target->type(), + /*nullable=*/true, + arrow::KeyValueMetadata::Make(keys, values))); + return Status::OK(); +} + +Result> VariantAccessBuilder::Build( + const std::string& field_name) const { + if (impl_->fields.empty()) { + return Status::Invalid("a variant-access projection needs at least one field"); + } + auto access_field = + arrow::field(field_name, arrow::struct_(impl_->fields), /*nullable=*/true, + arrow::KeyValueMetadata::Make({VariantDefs::kExtensionTypeKey}, + {VariantDefs::kExtensionTypeValue})); + auto field = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*access_field, field.get())); + return field; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_access_utils.cpp b/src/paimon/common/data/variant/variant_access_utils.cpp new file mode 100644 index 00000000..78180d1c --- /dev/null +++ b/src/paimon/common/data/variant/variant_access_utils.cpp @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_access_utils.h" + +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/types/data_field.h" + +namespace paimon { + +namespace { + +std::string GetDescription(const std::shared_ptr& field) { + if (field->metadata() == nullptr) { + return std::string(); + } + auto result = field->metadata()->Get(DataField::DESCRIPTION); + if (!result.ok()) { + return std::string(); + } + return result.ValueOrDie(); +} + +// Parses the description body from the right: the last two delimited tokens are `failOnError` +// and `timeZoneId` (neither contains the delimiter), and everything before them is the path, +// which may itself contain the delimiter inside object keys (e.g. `$['a;b']`). +std::vector SplitDescription(const std::string& description) { + std::string body = description.substr(sizeof(VariantAccessUtils::kMetadataKey) - 1); + size_t tz_sep = body.rfind(VariantAccessUtils::kDelimiter); + if (tz_sep == std::string::npos) { + return {body}; + } + size_t fail_sep = + tz_sep == 0 ? std::string::npos : body.rfind(VariantAccessUtils::kDelimiter, tz_sep - 1); + if (fail_sep == std::string::npos) { + return {body.substr(0, tz_sep), body.substr(tz_sep + 1)}; + } + return {body.substr(0, fail_sep), body.substr(fail_sep + 1, tz_sep - fail_sep - 1), + body.substr(tz_sep + 1)}; +} + +bool HasAccessDescription(const std::shared_ptr& field) { + return GetDescription(field).rfind(VariantAccessUtils::kMetadataKey, 0) == 0; +} + +} // namespace + +constexpr char VariantAccessUtils::kMetadataKey[]; +constexpr char VariantAccessUtils::kDelimiter; + +std::string VariantAccessUtils::BuildVariantMetadata(const std::string& path, bool fail_on_error, + const std::string& zone_id) { + return fmt::format("{}{}{}{}{}{}", kMetadataKey, path, kDelimiter, + fail_on_error ? "true" : "false", kDelimiter, zone_id); +} + +bool VariantAccessUtils::IsVariantAccessType(const std::shared_ptr& type) { + if (type == nullptr || type->id() != arrow::Type::STRUCT || type->num_fields() == 0) { + return false; + } + for (const auto& child : type->fields()) { + if (!HasAccessDescription(child)) { + return false; + } + } + return true; +} + +Result> VariantAccessUtils::ParseAccessSpecs( + const std::shared_ptr& access_field) { + if (!IsVariantAccessType(access_field->type())) { + return Status::Invalid( + fmt::format("field '{}' is not a variant-access projection", access_field->name())); + } + std::vector specs; + specs.reserve(access_field->type()->num_fields()); + for (const auto& child : access_field->type()->fields()) { + std::string description = GetDescription(child); + std::vector parts = SplitDescription(description); + if (parts.size() != 3) { + return Status::Invalid( + fmt::format("malformed variant access description '{}' on field '{}'", description, + child->name())); + } + VariantAccessSpec spec; + spec.path = parts[0]; + PAIMON_ASSIGN_OR_RAISE(spec.segments, VariantPathSegment::Parse(spec.path)); + spec.cast_args.fail_on_error = parts[1] == "true"; + spec.cast_args.zone_id = parts[2]; + spec.target_field = child; + specs.push_back(std::move(spec)); + } + return specs; +} + +Result> VariantAccessUtils::ClipShreddedFileField( + const std::vector& specs, const std::shared_ptr& file_field) { + if (file_field->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("variant file field '{}' is not a struct", file_field->name())); + } + const auto& file_struct = static_cast(*file_field->type()); + std::shared_ptr typed_value = + file_struct.GetFieldByName(VariantDefs::kTypedValueFieldName); + if (typed_value == nullptr) { + // The file stores the column unshredded; there is nothing to prune. + return file_field; + } + + bool can_clip = true; + std::set fields_to_read; + for (const auto& spec : specs) { + if (spec.segments.empty()) { + // A root path needs the whole variant. + can_clip = false; + break; + } + if (spec.segments[0].kind == VariantPathSegment::Kind::kObjectExtraction) { + // Only top-level object keys are pruned; nested paths still narrow to their + // top-level key. + fields_to_read.insert(spec.segments[0].key); + } else { + can_clip = false; + break; + } + } + if (!can_clip) { + return file_field; + } + + std::shared_ptr metadata_field = + file_struct.GetFieldByName(VariantDefs::kMetadataFieldName); + std::shared_ptr value_field = + file_struct.GetFieldByName(VariantDefs::kValueFieldName); + if (metadata_field == nullptr) { + return Status::Invalid( + fmt::format("shredded variant field '{}' misses metadata", file_field->name())); + } + + arrow::FieldVector typed_fields; + if (typed_value->type()->id() == arrow::Type::STRUCT) { + for (const auto& typed_child : typed_value->type()->fields()) { + if (fields_to_read.erase(typed_child->name()) > 0) { + typed_fields.push_back(typed_child); + } + } + } + + arrow::FieldVector clipped_fields = {metadata_field}; + if (!fields_to_read.empty() && value_field != nullptr) { + // Some requested key is not shredded; keep `value` for the binary fallback. + clipped_fields.push_back(value_field); + } + if (!typed_fields.empty()) { + clipped_fields.push_back(typed_value->WithType(arrow::struct_(typed_fields))); + } + return file_field->WithType(arrow::struct_(clipped_fields)); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_access_utils.h b/src/paimon/common/data/variant/variant_access_utils.h new file mode 100644 index 00000000..2cd79e9b --- /dev/null +++ b/src/paimon/common/data/variant/variant_access_utils.h @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/data/variant.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class Field; +} // namespace arrow + +namespace paimon { + +/// One extracted field of a variant-access projection: the extraction path, cast behavior, and +/// the target field the extracted value is cast to. +struct VariantAccessSpec { + std::string path; + std::vector segments; + VariantCastArgs cast_args; + std::shared_ptr target_field; +}; + +/// Utilities for variant-access projections: a variant column read as a struct whose children +/// each carry a `__VARIANT_METADATA;;` description (mirroring the +/// Java `VariantMetadataUtils`). Such a projection extracts the described paths from the variant +/// column at read time, reading only the required shredded sub-columns. +class VariantAccessUtils { + public: + static constexpr char kMetadataKey[] = "__VARIANT_METADATA"; + static constexpr char kDelimiter = ';'; + + VariantAccessUtils() = delete; + ~VariantAccessUtils() = delete; + + /// Builds the description string encoding one access spec. + static std::string BuildVariantMetadata(const std::string& path, bool fail_on_error, + const std::string& zone_id); + + /// Whether `type` is a variant-access projection: a struct with at least one field whose + /// children all carry a `__VARIANT_METADATA` description. + static bool IsVariantAccessType(const std::shared_ptr& type); + + /// Parses the access specs of a variant-access projection field. + static Result> ParseAccessSpecs( + const std::shared_ptr& access_field); + + /// Prunes a shredded file field down to the sub-columns required by the access specs: + /// `metadata` is always kept, `typed_value` is narrowed to the requested top-level keys, and + /// `value` is kept only when some requested key is not shredded. Returns `file_field` + /// unchanged when the file is unshredded or the paths cannot be pruned (root or array-first + /// paths). + static Result> ClipShreddedFileField( + const std::vector& specs, + const std::shared_ptr& file_field); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_access_utils_test.cpp b/src/paimon/common/data/variant/variant_access_utils_test.cpp new file mode 100644 index 00000000..6c9803d9 --- /dev/null +++ b/src/paimon/common/data/variant/variant_access_utils_test.cpp @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_access_utils.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +// An Arrow field carrying a `paimon.description` metadata entry. +std::shared_ptr DescribedField(const std::string& name, + const std::shared_ptr& type, + const std::string& description) { + return arrow::field(name, type, /*nullable=*/true, + arrow::key_value_metadata({DataField::DESCRIPTION}, {description})); +} + +std::shared_ptr AccessProjection( + const std::vector>& children) { + return arrow::field("v", arrow::struct_(children)); +} + +// A shredded file field: struct{[metadata], value, typed_value: struct{}}. +std::shared_ptr ShreddedFile( + const std::vector>& typed_children, bool with_metadata = true) { + arrow::FieldVector fields; + if (with_metadata) { + fields.push_back( + arrow::field(std::string(VariantDefs::kMetadataFieldName), arrow::binary())); + } + fields.push_back(arrow::field(std::string(VariantDefs::kValueFieldName), arrow::binary())); + fields.push_back(arrow::field(std::string(VariantDefs::kTypedValueFieldName), + arrow::struct_(typed_children))); + return arrow::field("v", arrow::struct_(fields)); +} + +std::vector ObjectKeySpecs() { + auto proj = AccessProjection({DescribedField( + "a", arrow::int32(), VariantAccessUtils::BuildVariantMetadata("$.a", false, "UTC"))}); + auto result = VariantAccessUtils::ParseAccessSpecs(proj); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); +} + +} // namespace + +TEST(VariantAccessUtilsTest, ParseAccessSpecsRejectsNonProjection) { + // A plain struct without access descriptions is not a variant-access projection. + auto plain = arrow::field("v", arrow::struct_({arrow::field("x", arrow::int32())})); + ASSERT_FALSE(VariantAccessUtils::IsVariantAccessType(plain->type())); + ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(plain)); +} + +TEST(VariantAccessUtilsTest, ParseAccessSpecsRejectsMalformedDescription) { + const std::string key = VariantAccessUtils::kMetadataKey; + // A description with no delimiter splits into a single part. + auto no_delim = AccessProjection({DescribedField("a", arrow::utf8(), key + "$.a")}); + ASSERT_TRUE(VariantAccessUtils::IsVariantAccessType(no_delim->type())); + ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(no_delim)); + // A single delimiter splits into two parts (still not the three that a spec needs). + auto one_delim = AccessProjection({DescribedField("a", arrow::utf8(), key + "$.a;true")}); + ASSERT_TRUE(VariantAccessUtils::IsVariantAccessType(one_delim->type())); + ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(one_delim)); +} + +TEST(VariantAccessUtilsTest, ParseAccessSpecsRoundTripsBuildMetadata) { + auto proj = AccessProjection({DescribedField( + "a", arrow::int32(), VariantAccessUtils::BuildVariantMetadata("$.a", true, "+08:00"))}); + ASSERT_OK_AND_ASSIGN(std::vector specs, + VariantAccessUtils::ParseAccessSpecs(proj)); + ASSERT_EQ(specs.size(), 1); + EXPECT_EQ(specs[0].path, "$.a"); + EXPECT_TRUE(specs[0].cast_args.fail_on_error); + EXPECT_EQ(specs[0].cast_args.zone_id, "+08:00"); +} + +TEST(VariantAccessUtilsTest, ClipRejectsNonStructFileField) { + auto non_struct = arrow::field("v", arrow::int32()); + ASSERT_NOK(VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), non_struct)); +} + +TEST(VariantAccessUtilsTest, ClipUnprunablePathsReturnFileUnchanged) { + auto file = ShreddedFile({arrow::field("a", arrow::int32())}); + + // A root path needs the whole variant, so the file field is returned unchanged. + auto root_proj = AccessProjection({DescribedField( + "r", arrow::utf8(), VariantAccessUtils::BuildVariantMetadata("$", false, "UTC"))}); + ASSERT_OK_AND_ASSIGN(std::vector root_specs, + VariantAccessUtils::ParseAccessSpecs(root_proj)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr root_clipped, + VariantAccessUtils::ClipShreddedFileField(root_specs, file)); + EXPECT_EQ(root_clipped, file); + + // An array-first path likewise cannot be pruned to a top-level key. + auto array_proj = AccessProjection({DescribedField( + "e", arrow::utf8(), VariantAccessUtils::BuildVariantMetadata("$[0]", false, "UTC"))}); + ASSERT_OK_AND_ASSIGN(std::vector array_specs, + VariantAccessUtils::ParseAccessSpecs(array_proj)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array_clipped, + VariantAccessUtils::ClipShreddedFileField(array_specs, file)); + EXPECT_EQ(array_clipped, file); +} + +TEST(VariantAccessUtilsTest, ClipUnshreddedFileFieldReturnedUnchanged) { + // No typed_value column: there is nothing to prune. + auto unshredded = arrow::field( + "v", + arrow::struct_({arrow::field(std::string(VariantDefs::kMetadataFieldName), arrow::binary()), + arrow::field(std::string(VariantDefs::kValueFieldName), arrow::binary())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr clipped, + VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), unshredded)); + EXPECT_EQ(clipped, unshredded); +} + +TEST(VariantAccessUtilsTest, ClipMissingMetadataFails) { + auto file_no_metadata = + ShreddedFile({arrow::field("a", arrow::int32())}, /*with_metadata=*/false); + ASSERT_NOK(VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), file_no_metadata)); +} + +TEST(VariantAccessUtilsTest, ClipNarrowsToRequestedKeys) { + // "$.a" is requested; the shredded typed_value keeps only "a" and drops the unrelated "b". + auto file = ShreddedFile({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr clipped, + VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), file)); + ASSERT_EQ(clipped->type()->id(), arrow::Type::STRUCT); + const auto& clipped_struct = static_cast(*clipped->type()); + // metadata is always kept; value is dropped because "a" is shredded; typed_value keeps "a". + ASSERT_NE(clipped_struct.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr); + EXPECT_EQ(clipped_struct.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed = clipped_struct.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed, nullptr); + ASSERT_EQ(typed->type()->num_fields(), 1); + EXPECT_EQ(typed->type()->field(0)->name(), "a"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_binary_util.cpp b/src/paimon/common/data/variant/variant_binary_util.cpp new file mode 100644 index 00000000..596d129c --- /dev/null +++ b/src/paimon/common/data/variant/variant_binary_util.cpp @@ -0,0 +1,599 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_binary_util.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" + +namespace paimon { + +int32_t VariantDecimal::Precision() const { + __uint128_t abs_value = + unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); + int32_t digits = 1; + while (abs_value >= 10) { + abs_value /= 10; + ++digits; + } + return digits; +} + +VariantDecimal VariantDecimal::StripTrailingZeros() const { + VariantDecimal result = *this; + if (result.unscaled == 0) { + result.scale = 0; + return result; + } + while (result.unscaled % 10 == 0) { + result.unscaled /= 10; + --result.scale; + } + return result; +} + +std::string VariantDecimal::ToPlainString() const { + __uint128_t abs_value = + unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); + std::string digits; + if (abs_value == 0) { + digits = "0"; + } else { + while (abs_value > 0) { + digits.push_back(static_cast('0' + static_cast(abs_value % 10))); + abs_value /= 10; + } + std::reverse(digits.begin(), digits.end()); + } + std::string result; + if (unscaled < 0) { + result.push_back('-'); + } + if (scale <= 0) { + result.append(digits); + result.append(static_cast(-scale), '0'); + } else if (static_cast(scale) < digits.size()) { + result.append(digits, 0, digits.size() - scale); + result.push_back('.'); + result.append(digits, digits.size() - scale, scale); + } else { + result.append("0."); + result.append(static_cast(scale) - digits.size(), '0'); + result.append(digits); + } + return result; +} + +Status VariantBinaryUtil::MalformedVariant(const std::string& message) { + if (message.empty()) { + return Status::Invalid("MALFORMED_VARIANT"); + } + return Status::Invalid(fmt::format("MALFORMED_VARIANT: {}", message)); +} + +Status VariantBinaryUtil::UnknownPrimitiveTypeInVariant(int32_t id) { + return Status::Invalid(fmt::format("UNKNOWN_PRIMITIVE_TYPE_IN_VARIANT, id: {}", id)); +} + +Status VariantBinaryUtil::VariantConstructorSizeLimit() { + return Status::Invalid("VARIANT_CONSTRUCTOR_SIZE_LIMIT"); +} + +Status VariantBinaryUtil::UnexpectedType(VariantValueType type) { + static constexpr const char* kTypeNames[] = { + "OBJECT", "ARRAY", "NULL", "BOOLEAN", "LONG", "STRING", "DOUBLE", + "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP_NTZ", "FLOAT", "BINARY", "UUID"}; + return Status::Invalid( + fmt::format("Expect type to be {}", kTypeNames[static_cast(type)])); +} + +Status VariantBinaryUtil::CheckIndex(int32_t pos, int32_t length) { + if (pos < 0 || pos >= length) { + return MalformedVariant( + fmt::format("index {} is out of bounds for a buffer of {} bytes", pos, length)); + } + return Status::OK(); +} + +void VariantBinaryUtil::WriteLong(int64_t value, int32_t num_bytes, uint8_t* bytes, int32_t pos) { + for (int32_t i = 0; i < num_bytes; ++i) { + bytes[pos + i] = static_cast((static_cast(value) >> (8 * i)) & 0xFF); + } +} + +Result VariantBinaryUtil::ReadLong(std::string_view bytes, int32_t pos, + int32_t num_bytes) { + auto length = static_cast(bytes.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); + uint64_t result = 0; + // All bytes except the most significant byte should be unsign-extended and shifted. The most + // significant byte should be sign-extended. + for (int32_t i = 0; i < num_bytes - 1; ++i) { + uint64_t unsigned_byte_value = static_cast(bytes[pos + i]); + result |= unsigned_byte_value << (8 * i); + } + int64_t signed_byte_value = static_cast(bytes[pos + num_bytes - 1]); + result |= static_cast(signed_byte_value) << (8 * (num_bytes - 1)); + return static_cast(result); +} + +Result VariantBinaryUtil::ReadUnsigned(std::string_view bytes, int32_t pos, + int32_t num_bytes) { + auto length = static_cast(bytes.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); + int64_t result = 0; + // Similar to the `ReadLong` loop, but all bytes should be unsign-extended. + for (int32_t i = 0; i < num_bytes; ++i) { + int64_t unsigned_byte_value = static_cast(bytes[pos + i]); + result |= unsigned_byte_value << (8 * i); + } + if (result < 0 || result > std::numeric_limits::max()) { + return MalformedVariant(fmt::format("unsigned value {} does not fit into int32", result)); + } + return static_cast(result); +} + +uint8_t VariantBinaryUtil::PrimitiveHeader(int32_t type) { + return static_cast(type << 2 | VariantDefs::kPrimitive); +} + +uint8_t VariantBinaryUtil::ShortStrHeader(int32_t size) { + return static_cast(size << 2 | VariantDefs::kShortStr); +} + +uint8_t VariantBinaryUtil::ObjectHeader(bool large_size, int32_t id_size, int32_t offset_size) { + return static_cast(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 4)) | + ((id_size - 1) << (VariantDefs::kBasicTypeBits + 2)) | + ((offset_size - 1) << VariantDefs::kBasicTypeBits) | + VariantDefs::kObject); +} + +uint8_t VariantBinaryUtil::ArrayHeader(bool large_size, int32_t offset_size) { + return static_cast(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 2)) | + ((offset_size - 1) << VariantDefs::kBasicTypeBits) | + VariantDefs::kArray); +} + +Result VariantBinaryUtil::GetTypeInfo(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + return (static_cast(value[pos]) >> VariantDefs::kBasicTypeBits) & + VariantDefs::kTypeInfoMask; +} + +Result VariantBinaryUtil::GetType(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + switch (basic_type) { + case VariantDefs::kShortStr: + return VariantValueType::kString; + case VariantDefs::kObject: + return VariantValueType::kObject; + case VariantDefs::kArray: + return VariantValueType::kArray; + default: + switch (type_info) { + case VariantDefs::kNull: + return VariantValueType::kNull; + case VariantDefs::kTrue: + case VariantDefs::kFalse: + return VariantValueType::kBoolean; + case VariantDefs::kInt1: + case VariantDefs::kInt2: + case VariantDefs::kInt4: + case VariantDefs::kInt8: + return VariantValueType::kLong; + case VariantDefs::kDouble: + return VariantValueType::kDouble; + case VariantDefs::kDecimal4: + case VariantDefs::kDecimal8: + case VariantDefs::kDecimal16: + return VariantValueType::kDecimal; + case VariantDefs::kDate: + return VariantValueType::kDate; + case VariantDefs::kTimestamp: + return VariantValueType::kTimestamp; + case VariantDefs::kTimestampNtz: + return VariantValueType::kTimestampNtz; + case VariantDefs::kFloat: + return VariantValueType::kFloat; + case VariantDefs::kBinary: + return VariantValueType::kBinary; + case VariantDefs::kLongStr: + return VariantValueType::kString; + case VariantDefs::kUuid: + return VariantValueType::kUuid; + default: + return UnknownPrimitiveTypeInVariant(type_info); + } + } +} + +Result VariantBinaryUtil::ValueSize(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + switch (basic_type) { + case VariantDefs::kShortStr: + return 1 + type_info; + case VariantDefs::kObject: { + PAIMON_ASSIGN_OR_RAISE(ObjectInfo info, GetObjectInfo(value, pos)); + PAIMON_ASSIGN_OR_RAISE( + int32_t data_size, + ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, + info.offset_size)); + return info.data_start - pos + data_size; + } + case VariantDefs::kArray: { + PAIMON_ASSIGN_OR_RAISE(ArrayInfo info, GetArrayInfo(value, pos)); + PAIMON_ASSIGN_OR_RAISE( + int32_t data_size, + ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, + info.offset_size)); + return info.data_start - pos + data_size; + } + default: + switch (type_info) { + case VariantDefs::kNull: + case VariantDefs::kTrue: + case VariantDefs::kFalse: + return 1; + case VariantDefs::kInt1: + return 2; + case VariantDefs::kInt2: + return 3; + case VariantDefs::kInt4: + case VariantDefs::kDate: + case VariantDefs::kFloat: + return 5; + case VariantDefs::kInt8: + case VariantDefs::kDouble: + case VariantDefs::kTimestamp: + case VariantDefs::kTimestampNtz: + return 9; + case VariantDefs::kDecimal4: + return 6; + case VariantDefs::kDecimal8: + return 10; + case VariantDefs::kDecimal16: + return 18; + case VariantDefs::kBinary: + case VariantDefs::kLongStr: { + PAIMON_ASSIGN_OR_RAISE(int32_t data_size, + ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + return 1 + VariantDefs::kU32Size + data_size; + } + case VariantDefs::kUuid: + return 17; + default: + return UnknownPrimitiveTypeInVariant(type_info); + } + } +} + +Result VariantBinaryUtil::GetBoolean(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || + (type_info != VariantDefs::kTrue && type_info != VariantDefs::kFalse)) { + return UnexpectedType(VariantValueType::kBoolean); + } + return type_info == VariantDefs::kTrue; +} + +Result VariantBinaryUtil::GetLong(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + constexpr const char* kExceptionMessage = "Expect type to be LONG/DATE/TIMESTAMP/TIMESTAMP_NTZ"; + if (basic_type != VariantDefs::kPrimitive) { + return Status::Invalid(kExceptionMessage); + } + switch (type_info) { + case VariantDefs::kInt1: + return ReadLong(value, pos + 1, 1); + case VariantDefs::kInt2: + return ReadLong(value, pos + 1, 2); + case VariantDefs::kInt4: + case VariantDefs::kDate: + return ReadLong(value, pos + 1, 4); + case VariantDefs::kInt8: + case VariantDefs::kTimestamp: + case VariantDefs::kTimestampNtz: + return ReadLong(value, pos + 1, 8); + default: + return Status::Invalid(kExceptionMessage); + } +} + +Result VariantBinaryUtil::GetDouble(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kDouble) { + return UnexpectedType(VariantValueType::kDouble); + } + PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 8)); + double result; + memcpy(&result, &bits, sizeof(result)); + return result; +} + +namespace { +// Checks whether the precision and scale of the decimal are within the limit. +Status CheckDecimal(const VariantDecimal& d, int32_t max_precision) { + if (d.Precision() > max_precision || d.scale > max_precision) { + return VariantBinaryUtil::MalformedVariant( + fmt::format("decimal precision {} or scale {} exceeds the maximum precision {}", + d.Precision(), d.scale, max_precision)); + } + return Status::OK(); +} +} // namespace + +Result VariantBinaryUtil::GetDecimalWithOriginalScale(std::string_view value, + int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive) { + return UnexpectedType(VariantValueType::kDecimal); + } + PAIMON_RETURN_NOT_OK(CheckIndex(pos + 1, length)); + // Interpret the scale byte as unsigned. If it is a negative byte, the unsigned value must be + // greater than `kMaxDecimal16Precision` and will trigger an error in `CheckDecimal`. + int32_t scale = static_cast(value[pos + 1]); + VariantDecimal result; + result.scale = scale; + switch (type_info) { + case VariantDefs::kDecimal4: { + PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 4)); + result.unscaled = unscaled; + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal4Precision)); + break; + } + case VariantDefs::kDecimal8: { + PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 8)); + result.unscaled = unscaled; + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal8Precision)); + break; + } + case VariantDefs::kDecimal16: { + PAIMON_RETURN_NOT_OK(CheckIndex(pos + 17, length)); + __uint128_t unscaled = 0; + for (int32_t i = 0; i < 16; ++i) { + unscaled |= static_cast<__uint128_t>(static_cast(value[pos + 2 + i])) + << (8 * i); + } + result.unscaled = static_cast<__int128_t>(unscaled); + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal16Precision)); + break; + } + default: + return UnexpectedType(VariantValueType::kDecimal); + } + return result; +} + +Result VariantBinaryUtil::GetDecimal(std::string_view value, int32_t pos) { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal decimal, GetDecimalWithOriginalScale(value, pos)); + return decimal.StripTrailingZeros(); +} + +Result VariantBinaryUtil::GetFloat(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kFloat) { + return UnexpectedType(VariantValueType::kFloat); + } + PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 4)); + auto int_bits = static_cast(bits); + float result; + memcpy(&result, &int_bits, sizeof(result)); + return result; +} + +Result VariantBinaryUtil::GetBinary(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kBinary) { + return UnexpectedType(VariantValueType::kBinary); + } + int32_t start = pos + 1 + VariantDefs::kU32Size; + PAIMON_ASSIGN_OR_RAISE(int32_t data_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + if (data_size > 0) { + PAIMON_RETURN_NOT_OK(CheckIndex(start + data_size - 1, length)); + } + return value.substr(start, data_size); +} + +Result VariantBinaryUtil::GetString(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type == VariantDefs::kShortStr || + (basic_type == VariantDefs::kPrimitive && type_info == VariantDefs::kLongStr)) { + int32_t start; + int32_t str_size; + if (basic_type == VariantDefs::kShortStr) { + start = pos + 1; + str_size = type_info; + } else { + start = pos + 1 + VariantDefs::kU32Size; + PAIMON_ASSIGN_OR_RAISE(str_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + } + if (str_size > 0) { + PAIMON_RETURN_NOT_OK(CheckIndex(start + str_size - 1, length)); + } + return value.substr(start, str_size); + } + return UnexpectedType(VariantValueType::kString); +} + +Result VariantBinaryUtil::GetUuid(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kUuid) { + return UnexpectedType(VariantValueType::kUuid); + } + int32_t start = pos + 1; + PAIMON_RETURN_NOT_OK(CheckIndex(start + 15, length)); + return value.substr(start, 16); +} + +std::string VariantBinaryUtil::UuidToString(std::string_view uuid_bytes) { + constexpr char kHexDigits[] = "0123456789abcdef"; + std::string result; + result.reserve(36); + for (size_t i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) { + result.push_back('-'); + } + auto byte = static_cast(uuid_bytes[i]); + result.push_back(kHexDigits[byte >> 4]); + result.push_back(kHexDigits[byte & 0xF]); + } + return result; +} + +Result VariantBinaryUtil::GetObjectInfo(std::string_view value, + int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kObject) { + return UnexpectedType(VariantValueType::kObject); + } + // Refer to the comment of the `VariantDefs::kObject` constant for the details of the object + // header encoding. Suppose `type_info` has a bit representation of 0_b4_b3b2_b1b0, the + // following line extracts b4 to determine whether the object uses a 1/4-byte size. + bool large_size = ((type_info >> 4) & 0x1) != 0; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + ObjectInfo info; + PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); + // Extracts b3b2 to determine the integer size of the field id list. + info.id_size = ((type_info >> 2) & 0x3) + 1; + // Extracts b1b0 to determine the integer size of the offset list. + info.offset_size = (type_info & 0x3) + 1; + info.id_start = pos + 1 + size_bytes; + // A corrupted variant can claim a near-INT32_MAX element count; compute the layout in + // 64-bit and bound it by the buffer before 32-bit arithmetic could overflow. + int64_t offset_start = static_cast(info.id_start) + + static_cast(info.num_elements) * info.id_size; + int64_t data_start = + offset_start + (static_cast(info.num_elements) + 1) * info.offset_size; + if (data_start > static_cast(value.size())) { + return MalformedVariant("object layout exceeds the value buffer"); + } + info.offset_start = static_cast(offset_start); + info.data_start = static_cast(data_start); + return info; +} + +Result VariantBinaryUtil::GetArrayInfo(std::string_view value, + int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kArray) { + return UnexpectedType(VariantValueType::kArray); + } + // Suppose `type_info` has a bit representation of 000_b2_b1b0, the following line extracts b2 + // to determine whether the array uses a 1/4-byte size. + bool large_size = ((type_info >> 2) & 0x1) != 0; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + ArrayInfo info; + PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); + // Extracts b1b0 to determine the integer size of the offset list. + info.offset_size = (type_info & 0x3) + 1; + info.offset_start = pos + 1 + size_bytes; + // See GetObjectInfo: bound the 64-bit layout by the buffer before 32-bit overflow. + int64_t data_start = static_cast(info.offset_start) + + (static_cast(info.num_elements) + 1) * info.offset_size; + if (data_start > static_cast(value.size())) { + return MalformedVariant("array layout exceeds the value buffer"); + } + info.data_start = static_cast(data_start); + return info; +} + +Result VariantBinaryUtil::GetMetadataKey(std::string_view metadata, int32_t id) { + auto length = static_cast(metadata.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(0, length)); + // Extracts the highest 2 bits in the metadata header to determine the integer size of the + // offset list. + int32_t offset_size = ((static_cast(metadata[0]) >> 6) & 0x3) + 1; + PAIMON_ASSIGN_OR_RAISE(int32_t dict_size, ReadUnsigned(metadata, 1, offset_size)); + if (id >= dict_size) { + return MalformedVariant(fmt::format( + "metadata key id {} is out of bounds for a dictionary of {} keys", id, dict_size)); + } + // There are a header byte, a `dict_size` with `offset_size` bytes, and `(dict_size + 1)` + // offsets before the string data. + // Bound the offset-table layout in 64-bit before 32-bit arithmetic could overflow on a + // corrupted dictionary size. + int64_t string_start64 = 1 + (static_cast(dict_size) + 2) * offset_size; + if (string_start64 > static_cast(length)) { + return MalformedVariant("metadata dictionary layout exceeds the metadata buffer"); + } + auto string_start = static_cast(string_start64); + PAIMON_ASSIGN_OR_RAISE(int32_t offset, + ReadUnsigned(metadata, 1 + (id + 1) * offset_size, offset_size)); + PAIMON_ASSIGN_OR_RAISE(int32_t next_offset, + ReadUnsigned(metadata, 1 + (id + 2) * offset_size, offset_size)); + if (offset > next_offset) { + return MalformedVariant("metadata key offsets are not monotonic"); + } + if (static_cast(string_start) + next_offset > static_cast(length)) { + return MalformedVariant("metadata key data exceeds the metadata buffer"); + } + return metadata.substr(string_start + offset, next_offset - offset); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_binary_util.h b/src/paimon/common/data/variant/variant_binary_util.h new file mode 100644 index 00000000..01a55f4b --- /dev/null +++ b/src/paimon/common/data/variant/variant_binary_util.h @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// The value type of a variant value. It is determined by the header byte but not a 1:1 mapping +/// (for example, INT1/2/4/8 all map to `kLong`). +enum class VariantValueType { + kObject, + kArray, + kNull, + kBoolean, + kLong, + kString, + kDouble, + kDecimal, + kDate, + kTimestamp, + kTimestampNtz, + kFloat, + kBinary, + kUuid, +}; + +/// An arbitrary-precision (up to 38 digits) decimal value decoded from a variant binary. The +/// numeric value is `unscaled * 10^(-scale)`. Unlike `paimon::Decimal`, `scale` may become +/// negative after `StripTrailingZeros` (mirroring `java.math.BigDecimal`). +struct VariantDecimal { + __int128_t unscaled = 0; + int32_t scale = 0; + + /// Number of decimal digits in the unscaled value (a value of zero has precision 1). + int32_t Precision() const; + + /// Removes trailing zero digits from the unscaled value, increasing `10^-scale` accordingly. + VariantDecimal StripTrailingZeros() const; + + /// Plain (non-scientific) string representation, e.g. `-12.340` or `100`. + std::string ToPlainString() const; + + bool operator==(const VariantDecimal& other) const { + return unscaled == other.unscaled && scale == other.scale; + } +}; + +/// Static functions for manipulating variant binaries. See `VariantDefs` for the binary format. +class VariantBinaryUtil { + public: + VariantBinaryUtil() = delete; + ~VariantBinaryUtil() = delete; + + /// The decoded layout of a variant object value. + struct ObjectInfo { + /// Number of object fields. + int32_t num_elements; + /// The integer size of the field id list. + int32_t id_size; + /// The integer size of the offset list. + int32_t offset_size; + /// The starting index of the field id list in the variant value. + int32_t id_start; + /// The starting index of the offset list in the variant value. + int32_t offset_start; + /// The starting index of field data in the variant value. + int32_t data_start; + }; + + /// The decoded layout of a variant array value. + struct ArrayInfo { + /// Number of array elements. + int32_t num_elements; + /// The integer size of the offset list. + int32_t offset_size; + /// The starting index of the offset list in the variant value. + int32_t offset_start; + /// The starting index of element data in the variant value. + int32_t data_start; + }; + + /// Creates the MALFORMED_VARIANT error. `message` describes the specific corruption for + /// debugging and is appended to the error when non-empty. + static Status MalformedVariant(const std::string& message = ""); + static Status UnknownPrimitiveTypeInVariant(int32_t id); + static Status VariantConstructorSizeLimit(); + static Status UnexpectedType(VariantValueType type); + + /// Checks the validity of an index `pos` in a buffer of `length` bytes. Returns + /// `MALFORMED_VARIANT` if it is out of bound. + static Status CheckIndex(int32_t pos, int32_t length); + + /// Writes the least significant `num_bytes` bytes in `value` into + /// `bytes[pos, pos + num_bytes)` in little endian. + static void WriteLong(int64_t value, int32_t num_bytes, uint8_t* bytes, int32_t pos); + + /// Reads a little-endian signed long value from `bytes[pos, pos + num_bytes)`. + static Result ReadLong(std::string_view bytes, int32_t pos, int32_t num_bytes); + + /// Reads a little-endian unsigned int value from `bytes[pos, pos + num_bytes)`. The value + /// must fit into a non-negative int32. + static Result ReadUnsigned(std::string_view bytes, int32_t pos, int32_t num_bytes); + + /// Adds a buffer-relative element `offset` to `base` in 64-bit and validates the result + /// stays inside `buffer_size`, guarding the 32-bit addition against corrupted offsets. + static Result CheckedElementPos(int32_t base, int32_t offset, size_t buffer_size) { + int64_t pos = static_cast(base) + offset; + if (pos >= static_cast(buffer_size)) { + return MalformedVariant("element offset points outside the value buffer"); + } + return static_cast(pos); + } + + static uint8_t PrimitiveHeader(int32_t type); + static uint8_t ShortStrHeader(int32_t size); + static uint8_t ObjectHeader(bool large_size, int32_t id_size, int32_t offset_size); + static uint8_t ArrayHeader(bool large_size, int32_t offset_size); + + /// Gets the type info bits from the variant value `value[pos...]`. + static Result GetTypeInfo(std::string_view value, int32_t pos); + + /// Gets the value type of the variant value `value[pos...]`. It is only legal to call `Get*` + /// if `GetType` returns the corresponding type (for example, it is only legal to call + /// `GetLong` if `GetType` returns `kLong`). + static Result GetType(std::string_view value, int32_t pos); + + /// Computes the size in bytes of the variant value `value[pos...]`. `value.size() - pos` is + /// an upper bound of the size, but the actual size can be smaller. + static Result ValueSize(std::string_view value, int32_t pos); + + static Result GetBoolean(std::string_view value, int32_t pos); + + /// Gets a long value from the variant value `value[pos...]`. It is only legal to call it if + /// `GetType` returns one of `kLong/kDate/kTimestamp/kTimestampNtz`. If the type is `kDate`, + /// the return value is guaranteed to fit into an int32 and represents the number of days from + /// the Unix epoch. If the type is `kTimestamp/kTimestampNtz`, the return value represents the + /// number of microseconds from the Unix epoch. + static Result GetLong(std::string_view value, int32_t pos); + + static Result GetDouble(std::string_view value, int32_t pos); + + /// Gets a decimal value from the variant value `value[pos...]`, keeping the stored scale. + static Result GetDecimalWithOriginalScale(std::string_view value, int32_t pos); + + /// Gets a decimal value from the variant value `value[pos...]` with trailing zeros stripped. + static Result GetDecimal(std::string_view value, int32_t pos); + + static Result GetFloat(std::string_view value, int32_t pos); + + /// Gets a binary value from the variant value `value[pos...]`. The returned view aliases + /// `value` and remains valid only as long as the underlying buffer. + static Result GetBinary(std::string_view value, int32_t pos); + + /// Gets a string value from the variant value `value[pos...]`. The returned view aliases + /// `value` and remains valid only as long as the underlying buffer. + static Result GetString(std::string_view value, int32_t pos); + + /// Gets a UUID value (16 bytes, big-endian) from the variant value `value[pos...]`. The + /// returned view aliases `value`. + static Result GetUuid(std::string_view value, int32_t pos); + + /// Formats a 16-byte big-endian UUID as the canonical lower-case string, e.g. + /// `123e4567-e89b-12d3-a456-426614174000`. + static std::string UuidToString(std::string_view uuid_bytes); + + /// Decodes the layout of the variant object value `value[pos...]`. + static Result GetObjectInfo(std::string_view value, int32_t pos); + + /// Decodes the layout of the variant array value `value[pos...]`. + static Result GetArrayInfo(std::string_view value, int32_t pos); + + /// Gets the key at `id` in the variant metadata. An out-of-bound `id` is considered a + /// malformed variant because it is read from the corresponding variant value. + static Result GetMetadataKey(std::string_view metadata, int32_t id); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_builder.cpp b/src/paimon/common/data/variant/variant_builder.cpp new file mode 100644 index 00000000..b3cf7ee5 --- /dev/null +++ b/src/paimon/common/data/variant/variant_builder.cpp @@ -0,0 +1,654 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_builder.h" + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "rapidjson/error/en.h" +#include "rapidjson/memorystream.h" +#include "rapidjson/reader.h" + +namespace paimon { + +namespace { + +// A rapidjson SAX handler that feeds parsed JSON events into a `VariantBuilder`, mirroring +// `GenericVariantBuilder.buildJson` in the Java implementation. Numbers are delivered as raw +// text (`kParseNumbersAsStringsFlag`) so that exact decimal semantics match Jackson's. +class JsonToVariantHandler + : public rapidjson::BaseReaderHandler, JsonToVariantHandler> { + public: + explicit JsonToVariantHandler(VariantBuilder* builder) : builder_(builder) {} + + bool Null() { + BeforeValue(); + return Ok(builder_->AppendNull()); + } + + bool Bool(bool b) { + BeforeValue(); + return Ok(builder_->AppendBoolean(b)); + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool /*copy*/) { + BeforeValue(); + return Ok(AppendNumber(std::string_view(str, length))); + } + + bool String(const char* str, rapidjson::SizeType length, bool /*copy*/) { + BeforeValue(); + return Ok(builder_->AppendString(std::string_view(str, length))); + } + + bool StartObject() { + BeforeValue(); + contexts_.emplace_back(Context{true, builder_->GetWritePos(), {}, {}, {}}); + return true; + } + + bool Key(const char* str, rapidjson::SizeType length, bool /*copy*/) { + contexts_.back().pending_key.assign(str, length); + return true; + } + + bool EndObject(rapidjson::SizeType /*member_count*/) { + Context context = std::move(contexts_.back()); + contexts_.pop_back(); + return Ok(builder_->FinishWritingObject(context.start, &context.fields)); + } + + bool StartArray() { + BeforeValue(); + contexts_.emplace_back(Context{false, builder_->GetWritePos(), {}, {}, {}}); + return true; + } + + bool EndArray(rapidjson::SizeType /*element_count*/) { + Context context = std::move(contexts_.back()); + contexts_.pop_back(); + return Ok(builder_->FinishWritingArray(context.start, context.offsets)); + } + + const Status& status() const { + return status_; + } + + private: + struct Context { + bool is_object; + int32_t start; + std::vector fields; + std::vector offsets; + std::string pending_key; + }; + + bool Ok(const Status& status) { + if (!status.ok()) { + status_ = status; + return false; + } + return true; + } + + // Records the offset of the value that is about to be appended in the enclosing container. + void BeforeValue() { + if (contexts_.empty()) { + return; + } + Context& top = contexts_.back(); + int32_t offset = builder_->GetWritePos() - top.start; + if (top.is_object) { + int32_t id = builder_->AddKey(top.pending_key); + top.fields.emplace_back(top.pending_key, id, offset); + } else { + top.offsets.push_back(offset); + } + } + + // Mirrors the Java number handling: integers that fit in a long are appended as long; + // everything else is first tried as an exact decimal and falls back to double. + Status AppendNumber(std::string_view text) { + bool integral = true; + for (char c : text) { + if (c != '-' && !(c >= '0' && c <= '9')) { + integral = false; + break; + } + } + if (integral) { + int64_t long_value = 0; + auto [ptr, ec] = std::from_chars(text.data(), text.data() + text.size(), long_value); + if (ec == std::errc() && ptr == text.data() + text.size()) { + return builder_->AppendLong(long_value); + } + } + PAIMON_ASSIGN_OR_RAISE(bool appended, TryAppendDecimal(text)); + if (appended) { + return Status::OK(); + } + char* end = nullptr; + std::string text_copy(text); + double double_value = std::strtod(text_copy.c_str(), &end); + if (end != text_copy.c_str() + text_copy.size()) { + return Status::Invalid(fmt::format("Invalid JSON number: {}", text)); + } + return builder_->AppendDouble(double_value); + } + + // Tries to append a JSON number as an exact decimal. Returns whether it succeeded. The input + // must only use the decimal format (an integer value with an optional '.' in it) and must not + // use scientific notation. It also must fit into the precision limitation of decimal types. + Result TryAppendDecimal(std::string_view text) { + for (char c : text) { + if (c != '-' && c != '.' && !(c >= '0' && c <= '9')) { + return false; + } + } + bool negative = false; + size_t i = 0; + if (i < text.size() && text[i] == '-') { + negative = true; + ++i; + } + __int128_t unscaled = 0; + int32_t scale = 0; + int32_t significant_digits = 0; + bool seen_point = false; + bool seen_nonzero = false; + for (; i < text.size(); ++i) { + char c = text[i]; + if (c == '.') { + seen_point = true; + continue; + } + if (seen_point) { + ++scale; + } + if (c != '0' || seen_nonzero) { + seen_nonzero = true; + ++significant_digits; + } + if (significant_digits > VariantDefs::kMaxDecimal16Precision) { + return false; + } + unscaled = unscaled * 10 + (c - '0'); + } + if (scale > VariantDefs::kMaxDecimal16Precision) { + return false; + } + VariantDecimal decimal; + decimal.unscaled = negative ? -unscaled : unscaled; + decimal.scale = scale; + PAIMON_RETURN_NOT_OK(builder_->AppendDecimal(decimal)); + return true; + } + + VariantBuilder* builder_; + std::vector contexts_; + Status status_; +}; + +} // namespace + +Result> VariantBuilder::ParseJson( + std::string_view json, bool allow_duplicate_keys, const std::shared_ptr& pool) { + VariantBuilder builder(allow_duplicate_keys); + JsonToVariantHandler handler(&builder); + rapidjson::Reader reader; + rapidjson::MemoryStream stream(json.data(), json.size()); + rapidjson::ParseResult result = + reader.Parse(stream, handler); + if (!result) { + if (!handler.status().ok()) { + return handler.status(); + } + return Status::Invalid(fmt::format("Failed to parse JSON: {} (at offset {})", + rapidjson::GetParseError_En(result.Code()), + result.Offset())); + } + return builder.Build(pool); +} + +Result> VariantBuilder::Build( + const std::shared_ptr& pool) { + auto num_keys = static_cast(dictionary_keys_.size()); + // Use int64 to avoid overflow in accumulating lengths. + int64_t dictionary_string_size = 0; + for (const std::string& key : dictionary_keys_) { + dictionary_string_size += static_cast(key.size()); + } + // Determine the number of bytes required per offset entry. The largest offset is the + // one-past-the-end value, which is the total string size. It's very unlikely that the number + // of keys could be larger, but incorporate that into the calculation in case of pathological + // data. + int64_t max_size = std::max(dictionary_string_size, static_cast(num_keys)); + if (max_size > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + int32_t offset_size = GetIntegerSize(static_cast(max_size)); + + int32_t offset_start = 1 + offset_size; + int32_t string_start = offset_start + (num_keys + 1) * offset_size; + int64_t metadata_size = string_start + dictionary_string_size; + if (metadata_size > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + + std::shared_ptr metadata = + Bytes::AllocateBytes(static_cast(metadata_size), pool.get()); + auto* metadata_data = reinterpret_cast(metadata->data()); + int32_t header_byte = VariantDefs::kVersion | ((offset_size - 1) << 6); + VariantBinaryUtil::WriteLong(header_byte, 1, metadata_data, 0); + VariantBinaryUtil::WriteLong(num_keys, offset_size, metadata_data, 1); + int32_t current_offset = 0; + for (int32_t i = 0; i < num_keys; ++i) { + VariantBinaryUtil::WriteLong(current_offset, offset_size, metadata_data, + offset_start + i * offset_size); + const std::string& key = dictionary_keys_[i]; + memcpy(metadata_data + string_start + current_offset, key.data(), key.size()); + current_offset += static_cast(key.size()); + } + VariantBinaryUtil::WriteLong(current_offset, offset_size, metadata_data, + offset_start + num_keys * offset_size); + + std::shared_ptr value = + Bytes::AllocateBytes(static_cast(write_pos_), pool.get()); + memcpy(value->data(), write_buffer_.data(), static_cast(write_pos_)); + return GenericVariant::Create(std::move(value), std::move(metadata)); +} + +Status VariantBuilder::AppendString(std::string_view str) { + bool long_str = static_cast(str.size()) > VariantDefs::kMaxShortStrSize; + PAIMON_RETURN_NOT_OK(CheckCapacity((long_str ? 1 + VariantDefs::kU32Size : 1) + + static_cast(str.size()))); + if (long_str) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kLongStr); + VariantBinaryUtil::WriteLong(static_cast(str.size()), VariantDefs::kU32Size, + write_buffer_.data(), write_pos_); + write_pos_ += VariantDefs::kU32Size; + } else { + write_buffer_[write_pos_++] = + VariantBinaryUtil::ShortStrHeader(static_cast(str.size())); + } + memcpy(write_buffer_.data() + write_pos_, str.data(), str.size()); + write_pos_ += static_cast(str.size()); + return Status::OK(); +} + +Status VariantBuilder::AppendNull() { + PAIMON_RETURN_NOT_OK(CheckCapacity(1)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kNull); + return Status::OK(); +} + +Status VariantBuilder::AppendBoolean(bool b) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1)); + write_buffer_[write_pos_++] = + VariantBinaryUtil::PrimitiveHeader(b ? VariantDefs::kTrue : VariantDefs::kFalse); + return Status::OK(); +} + +Status VariantBuilder::AppendLong(int64_t l) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt1); + VariantBinaryUtil::WriteLong(l, 1, write_buffer_.data(), write_pos_); + write_pos_ += 1; + } else if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt2); + VariantBinaryUtil::WriteLong(l, 2, write_buffer_.data(), write_pos_); + write_pos_ += 2; + } else if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt4); + VariantBinaryUtil::WriteLong(l, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + } else { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt8); + VariantBinaryUtil::WriteLong(l, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + } + return Status::OK(); +} + +Status VariantBuilder::AppendDouble(double d) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDouble); + int64_t bits; + memcpy(&bits, &d, sizeof(bits)); + VariantBinaryUtil::WriteLong(bits, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendDecimal(const VariantDecimal& d) { + PAIMON_RETURN_NOT_OK(CheckCapacity(2 + 16)); + int32_t precision = d.Precision(); + if (d.scale < 0 || d.scale > VariantDefs::kMaxDecimal16Precision || + precision > VariantDefs::kMaxDecimal16Precision) { + return Status::Invalid( + fmt::format("Decimal precision {} and scale {} must fit into the variant decimal " + "limit {}", + precision, d.scale, VariantDefs::kMaxDecimal16Precision)); + } + if (d.scale <= VariantDefs::kMaxDecimal4Precision && + precision <= VariantDefs::kMaxDecimal4Precision) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal4); + write_buffer_[write_pos_++] = static_cast(d.scale); + VariantBinaryUtil::WriteLong(static_cast(d.unscaled), 4, write_buffer_.data(), + write_pos_); + write_pos_ += 4; + } else if (d.scale <= VariantDefs::kMaxDecimal8Precision && + precision <= VariantDefs::kMaxDecimal8Precision) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal8); + write_buffer_[write_pos_++] = static_cast(d.scale); + VariantBinaryUtil::WriteLong(static_cast(d.unscaled), 8, write_buffer_.data(), + write_pos_); + write_pos_ += 8; + } else { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal16); + write_buffer_[write_pos_++] = static_cast(d.scale); + auto bits = static_cast<__uint128_t>(d.unscaled); + for (int32_t i = 0; i < 16; ++i) { + write_buffer_[write_pos_ + i] = static_cast((bits >> (8 * i)) & 0xFF); + } + write_pos_ += 16; + } + return Status::OK(); +} + +Status VariantBuilder::AppendDate(int32_t days_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 4)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDate); + VariantBinaryUtil::WriteLong(days_since_epoch, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + return Status::OK(); +} + +Status VariantBuilder::AppendTimestamp(int64_t micros_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kTimestamp); + VariantBinaryUtil::WriteLong(micros_since_epoch, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendTimestampNtz(int64_t micros_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kTimestampNtz); + VariantBinaryUtil::WriteLong(micros_since_epoch, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendFloat(float f) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 4)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kFloat); + int32_t bits; + memcpy(&bits, &f, sizeof(bits)); + VariantBinaryUtil::WriteLong(bits, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + return Status::OK(); +} + +Status VariantBuilder::AppendBinary(std::string_view binary) { + PAIMON_RETURN_NOT_OK( + CheckCapacity(1 + VariantDefs::kU32Size + static_cast(binary.size()))); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kBinary); + VariantBinaryUtil::WriteLong(static_cast(binary.size()), VariantDefs::kU32Size, + write_buffer_.data(), write_pos_); + write_pos_ += VariantDefs::kU32Size; + memcpy(write_buffer_.data() + write_pos_, binary.data(), binary.size()); + write_pos_ += static_cast(binary.size()); + return Status::OK(); +} + +Status VariantBuilder::AppendUuid(std::string_view uuid_bytes) { + if (uuid_bytes.size() != 16) { + return Status::Invalid("UUID must be 16 bytes"); + } + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 16)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kUuid); + // UUID is stored big-endian, so don't use WriteLong. + memcpy(write_buffer_.data() + write_pos_, uuid_bytes.data(), 16); + write_pos_ += 16; + return Status::OK(); +} + +int32_t VariantBuilder::AddKey(std::string_view key) { + auto it = dictionary_.find(std::string(key)); + if (it != dictionary_.end()) { + return it->second; + } + auto id = static_cast(dictionary_keys_.size()); + dictionary_.emplace(std::string(key), id); + dictionary_keys_.emplace_back(key); + return id; +} + +Status VariantBuilder::FinishWritingObject(int32_t start, std::vector* fields) { + auto size = static_cast(fields->size()); + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.key < b.key; }); + int32_t max_id = size == 0 ? 0 : (*fields)[0].id; + if (allow_duplicate_keys_) { + int32_t distinct_pos = 0; + // Maintain a list of distinct keys in-place. + for (int32_t i = 1; i < size; ++i) { + max_id = std::max(max_id, (*fields)[i].id); + if ((*fields)[i].id == (*fields)[i - 1].id) { + // Found a duplicate key. Keep the field with a greater offset. + if ((*fields)[distinct_pos].offset < (*fields)[i].offset) { + (*fields)[distinct_pos].offset = (*fields)[i].offset; + } + } else { + // Found a distinct key. Add the field to the list. + ++distinct_pos; + (*fields)[distinct_pos] = (*fields)[i]; + } + } + if (distinct_pos + 1 < size) { + size = distinct_pos + 1; + fields->erase(fields->begin() + size, fields->end()); + // Sort the fields by offsets so that we can move the value data of each field to the + // new offset without overwriting the fields after it. + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.offset < b.offset; }); + int32_t current_offset = 0; + for (int32_t i = 0; i < size; ++i) { + int32_t old_offset = (*fields)[i].offset; + PAIMON_ASSIGN_OR_RAISE( + int32_t field_size, + VariantBinaryUtil::ValueSize( + std::string_view(reinterpret_cast(write_buffer_.data()), + static_cast(write_pos_)), + start + old_offset)); + memmove(write_buffer_.data() + start + current_offset, + write_buffer_.data() + start + old_offset, static_cast(field_size)); + (*fields)[i].offset = current_offset; + current_offset += field_size; + } + write_pos_ = start + current_offset; + // Change back to the sort order by field keys to meet the variant spec. + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.key < b.key; }); + } + } else { + for (int32_t i = 1; i < size; ++i) { + max_id = std::max(max_id, (*fields)[i].id); + if ((*fields)[i].key == (*fields)[i - 1].key) { + return Status::Invalid("VARIANT_DUPLICATE_KEY"); + } + } + } + int32_t data_size = write_pos_ - start; + bool large_size = size > VariantDefs::kU8Max; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + int32_t id_size = GetIntegerSize(max_id); + int32_t offset_size = GetIntegerSize(data_size); + // The space for the header byte, object size, id list, and offset list. + int32_t header_size = 1 + size_bytes + size * id_size + (size + 1) * offset_size; + PAIMON_RETURN_NOT_OK(CheckCapacity(header_size)); + // Shift the just-written field data to make room for the object header section. + memmove(write_buffer_.data() + start + header_size, write_buffer_.data() + start, + static_cast(data_size)); + write_pos_ += header_size; + write_buffer_[start] = VariantBinaryUtil::ObjectHeader(large_size, id_size, offset_size); + VariantBinaryUtil::WriteLong(size, size_bytes, write_buffer_.data(), start + 1); + int32_t id_start = start + 1 + size_bytes; + int32_t offset_start = id_start + size * id_size; + for (int32_t i = 0; i < size; ++i) { + VariantBinaryUtil::WriteLong((*fields)[i].id, id_size, write_buffer_.data(), + id_start + i * id_size); + VariantBinaryUtil::WriteLong((*fields)[i].offset, offset_size, write_buffer_.data(), + offset_start + i * offset_size); + } + VariantBinaryUtil::WriteLong(data_size, offset_size, write_buffer_.data(), + offset_start + size * offset_size); + return Status::OK(); +} + +Status VariantBuilder::FinishWritingArray(int32_t start, const std::vector& offsets) { + int32_t data_size = write_pos_ - start; + auto size = static_cast(offsets.size()); + bool large_size = size > VariantDefs::kU8Max; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + int32_t offset_size = GetIntegerSize(data_size); + // The space for the header byte, array size, and offset list. + int32_t header_size = 1 + size_bytes + (size + 1) * offset_size; + PAIMON_RETURN_NOT_OK(CheckCapacity(header_size)); + // Shift the just-written element data to make room for the header section. + memmove(write_buffer_.data() + start + header_size, write_buffer_.data() + start, + static_cast(data_size)); + write_pos_ += header_size; + write_buffer_[start] = VariantBinaryUtil::ArrayHeader(large_size, offset_size); + VariantBinaryUtil::WriteLong(size, size_bytes, write_buffer_.data(), start + 1); + int32_t offset_start = start + 1 + size_bytes; + for (int32_t i = 0; i < size; ++i) { + VariantBinaryUtil::WriteLong(offsets[i], offset_size, write_buffer_.data(), + offset_start + i * offset_size); + } + VariantBinaryUtil::WriteLong(data_size, offset_size, write_buffer_.data(), + offset_start + size * offset_size); + return Status::OK(); +} + +Status VariantBuilder::AppendVariant(const GenericVariant& v) { + return AppendVariantImpl(v.RawValue(), v.Metadata(), v.Pos()); +} + +Status VariantBuilder::AppendVariantImpl(std::string_view value, std::string_view metadata, + int32_t pos) { + PAIMON_RETURN_NOT_OK(VariantBinaryUtil::CheckIndex(pos, static_cast(value.size()))); + int32_t basic_type = static_cast(value[pos]) & VariantDefs::kBasicTypeMask; + switch (basic_type) { + case VariantDefs::kObject: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(value, pos)); + std::vector fields; + fields.reserve(info.num_elements); + int32_t start = write_pos_; + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, + VariantBinaryUtil::ReadUnsigned( + value, info.id_start + info.id_size * i, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + int32_t element_pos = info.data_start + offset; + PAIMON_ASSIGN_OR_RAISE(std::string_view key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + int32_t new_id = AddKey(key); + fields.emplace_back(std::string(key), new_id, write_pos_ - start); + PAIMON_RETURN_NOT_OK(AppendVariantImpl(value, metadata, element_pos)); + } + return FinishWritingObject(start, &fields); + } + case VariantDefs::kArray: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(value, pos)); + std::vector offsets; + offsets.reserve(info.num_elements); + int32_t start = write_pos_; + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + int32_t element_pos = info.data_start + offset; + offsets.push_back(write_pos_ - start); + PAIMON_RETURN_NOT_OK(AppendVariantImpl(value, metadata, element_pos)); + } + return FinishWritingArray(start, offsets); + } + default: + return ShallowAppendVariant(value, pos); + } +} + +Status VariantBuilder::ShallowAppendVariant(std::string_view value, int32_t pos) { + PAIMON_ASSIGN_OR_RAISE(int32_t size, VariantBinaryUtil::ValueSize(value, pos)); + PAIMON_RETURN_NOT_OK( + VariantBinaryUtil::CheckIndex(pos + size - 1, static_cast(value.size()))); + PAIMON_RETURN_NOT_OK(CheckCapacity(size)); + memcpy(write_buffer_.data() + write_pos_, value.data() + pos, static_cast(size)); + write_pos_ += size; + return Status::OK(); +} + +Status VariantBuilder::CheckCapacity(int32_t additional) { + int32_t required = write_pos_ + additional; + if (required > static_cast(write_buffer_.size())) { + // Allocate a new buffer with a capacity of the next power of 2 of `required`. + auto new_capacity = static_cast(write_buffer_.size()); + while (new_capacity < required) { + new_capacity *= 2; + if (new_capacity > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + } + write_buffer_.resize(static_cast(new_capacity)); + } + return Status::OK(); +} + +int32_t VariantBuilder::GetIntegerSize(int32_t value) { + if (value <= VariantDefs::kU8Max) { + return 1; + } + if (value <= VariantDefs::kU16Max) { + return 2; + } + if (value <= VariantDefs::kU24Max) { + return 3; + } + return 4; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_builder.h b/src/paimon/common/data/variant/variant_builder.h new file mode 100644 index 00000000..4c959592 --- /dev/null +++ b/src/paimon/common/data/variant/variant_builder.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { + +/// Builds variant value and metadata binaries, either by parsing JSON values or by appending +/// values directly. +class VariantBuilder { + public: + /// Temporarily stores the information of a field. All fields of a JSON object are collected, + /// sorted by their keys, and then the variant object is built in sorted order. + struct FieldEntry { + std::string key; + int32_t id; + int32_t offset; + + FieldEntry(std::string key, int32_t id, int32_t offset) + : key(std::move(key)), id(id), offset(offset) {} + }; + + explicit VariantBuilder(bool allow_duplicate_keys) + : allow_duplicate_keys_(allow_duplicate_keys) {} + + /// Parses a JSON string as a variant value. + /// + /// When `allow_duplicate_keys` is true, the last occurrence of a duplicate object key wins; + /// otherwise duplicate keys make the parse fail. + static Result> ParseJson( + std::string_view json, bool allow_duplicate_keys, const std::shared_ptr& pool); + + /// Builds the variant metadata from the collected dictionary keys and returns the variant + /// result. + Result> Build(const std::shared_ptr& pool); + + /// The variant value written so far, without metadata. Used in shredding to produce a final + /// value where all shredded values refer to a common metadata. + std::string_view ValueWithoutMetadata() const { + return {reinterpret_cast(write_buffer_.data()), + static_cast(write_pos_)}; + } + + Status AppendString(std::string_view str); + Status AppendNull(); + Status AppendBoolean(bool b); + /// Appends a long value. The actual used integer type depends on the value range. + Status AppendLong(int64_t l); + Status AppendDouble(double d); + /// Appends a decimal value. Its precision and scale must fit into `kMaxDecimal16Precision`. + Status AppendDecimal(const VariantDecimal& d); + Status AppendDate(int32_t days_since_epoch); + Status AppendTimestamp(int64_t micros_since_epoch); + Status AppendTimestampNtz(int64_t micros_since_epoch); + Status AppendFloat(float f); + Status AppendBinary(std::string_view binary); + /// Appends a UUID value (16 bytes, big-endian). + Status AppendUuid(std::string_view uuid_bytes); + + /// Adds a key to the variant dictionary and returns its id. If the key already exists, the + /// dictionary is not modified. + int32_t AddKey(std::string_view key); + + /// The current write position of the variant builder. It is used together with + /// `FinishWritingObject` or `FinishWritingArray`. + int32_t GetWritePos() const { + return write_pos_; + } + + /// Finishes writing a variant object after all of its fields have already been written. The + /// process is as follows: + /// 1. The caller calls `GetWritePos` before writing any fields to obtain the `start` + /// parameter. + /// 2. The caller appends all the object fields to the builder. In the meantime, it should + /// maintain the `fields` parameter. Before appending each field, it should append an entry + /// to `fields` to record the offset of the field, computed as `GetWritePos() - start`. + /// 3. The caller calls `FinishWritingObject` to finish writing a variant object. + /// + /// This function sorts the fields by key. If there are duplicate field keys: + /// - when `allow_duplicate_keys` is true, the field with the greatest offset value (the last + /// appended one) is kept; + /// - otherwise, the call fails. + Status FinishWritingObject(int32_t start, std::vector* fields); + + /// Finishes writing a variant array after all of its elements have already been written. The + /// process is similar to that of `FinishWritingObject`. + Status FinishWritingArray(int32_t start, const std::vector& offsets); + + /// Appends a variant value. The keys of the input variant are inserted into the current + /// variant dictionary and the value is rebuilt with new field ids. For scalar values, the + /// binary slice is copied directly. + Status AppendVariant(const GenericVariant& v); + + /// Appends the variant value without rewriting or creating any metadata. This is used when + /// building an object during shredding, where there is a fixed pre-existing metadata that all + /// shredded values refer to. + Status ShallowAppendVariant(std::string_view value, int32_t pos); + + private: + Status CheckCapacity(int32_t additional); + Status AppendVariantImpl(std::string_view value, std::string_view metadata, int32_t pos); + static int32_t GetIntegerSize(int32_t value); + + // The write buffer in building the variant value. Its first `write_pos_` bytes have been + // written. + std::vector write_buffer_ = std::vector(128); + int32_t write_pos_ = 0; + // Maps keys to a monotonically increasing id. + std::unordered_map dictionary_; + // Stores all keys in `dictionary_` in the order of id. + std::vector dictionary_keys_; + const bool allow_duplicate_keys_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_defs.h b/src/paimon/common/data/variant/variant_defs.h new file mode 100644 index 00000000..a9bc8564 --- /dev/null +++ b/src/paimon/common/data/variant/variant_defs.h @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +enum class VariantShreddingInferenceMode { + PER_FILE, + ADAPTIVE, +}; + +/// Constants of the Paimon Variant type and the Variant Binary Encoding, which follows the +/// parquet-format VariantEncoding.md specification (compatible with the Java / Spark +/// implementation). +/// +/// A variant is made up of 2 binaries: value and metadata. A variant value consists of a one-byte +/// header and a number of content bytes (can be zero). The header byte is divided into upper 6 +/// bits (called "type info") and lower 2 bits (called "basic type"). +/// +/// The variant metadata includes a version id and a dictionary of distinct strings +/// (case-sensitive). Its binary format is: +/// - Version: 1-byte unsigned integer. The only acceptable value is 1 currently. +/// - Dictionary size: `offset_size`-byte little-endian unsigned integer. The number of keys in the +/// dictionary. +/// - Offsets: (size + 1) * `offset_size`-byte little-endian unsigned integers. `offsets[i]` +/// represents the starting position of string i, counting starting from the address of +/// `offsets[0]`. Strings must be stored contiguously, so we don't need to store the string size, +/// instead, we compute it with `offset[i + 1] - offset[i]`. +/// - UTF-8 string data. +/// +/// A Variant field uses `struct` as its +/// underlying physical storage in Apache Arrow Schema, and is marked as the Paimon Variant +/// extension type by attaching specific **KeyValueMetadata** on the outer field. +class VariantDefs { + public: + VariantDefs() = delete; + ~VariantDefs() = delete; + + /// Metadata key identifying a Paimon extension type field (shared with BLOB). + static constexpr char kExtensionTypeKey[] = "paimon.extension.type"; + /// Metadata value identifying a Paimon Variant extension type field. + static constexpr char kExtensionTypeValue[] = "paimon.type.variant"; + + /// Name of the binary child field holding the variant value. + static constexpr char kValueFieldName[] = "value"; + /// Name of the binary child field holding the variant metadata. + static constexpr char kMetadataFieldName[] = "metadata"; + /// Name of the typed child field of a shredded variant (parquet VariantShredding.md). + static constexpr char kTypedValueFieldName[] = "typed_value"; + /// Paimon field id of the `value` child field. + static constexpr int32_t kValueFieldId = 0; + /// Paimon field id of the `metadata` child field. + static constexpr int32_t kMetadataFieldId = 1; + + static constexpr int32_t kBasicTypeBits = 2; + static constexpr int32_t kBasicTypeMask = 0x3; + static constexpr int32_t kTypeInfoMask = 0x3F; + /// The inclusive maximum value of the type info value. It is the size limit of `kShortStr`. + static constexpr int32_t kMaxShortStrSize = 0x3F; + + /// Primitive value. The type info value must be one of the primitive type values below. + static constexpr int32_t kPrimitive = 0; + /// Short string value. The type info value is the string size, which must be in + /// `[0, kMaxShortStrSize]`. The string content bytes directly follow the header byte. + static constexpr int32_t kShortStr = 1; + /// Object value. The content contains a size, a list of field ids, a list of field offsets, + /// and the actual field data. The length of the id list is `size`, while the length of the + /// offset list is `size + 1`, where the last offset represents the total size of the field + /// data. The fields in an object must be sorted by the field name in alphabetical order. + /// Duplicate field names in one object are not allowed. + /// The type info is 0_b4_b3b2_b1b0 (MSB is 0), where: + /// - b4 specifies the type of size. When it is 0/1, `size` is a little-endian 1/4-byte + /// unsigned integer. + /// - b3b2/b1b0 specifies the integer type of id and offset. When the 2 bits are 0/1/2, the + /// list contains 1/2/3-byte little-endian unsigned integers. + static constexpr int32_t kObject = 2; + /// Array value. The content contains a size, a list of field offsets, and the actual element + /// data. It is similar to an object without the id list. The type info is 000_b2_b1b0: + /// - b2 specifies the type of size. + /// - b1b0 specifies the integer type of offset. + static constexpr int32_t kArray = 3; + + /// JSON null value. Empty content. + static constexpr int32_t kNull = 0; + /// True value. Empty content. + static constexpr int32_t kTrue = 1; + /// False value. Empty content. + static constexpr int32_t kFalse = 2; + /// 1-byte little-endian signed integer. + static constexpr int32_t kInt1 = 3; + /// 2-byte little-endian signed integer. + static constexpr int32_t kInt2 = 4; + /// 4-byte little-endian signed integer. + static constexpr int32_t kInt4 = 5; + /// 8-byte little-endian signed integer. + static constexpr int32_t kInt8 = 6; + /// 8-byte IEEE double. + static constexpr int32_t kDouble = 7; + /// 4-byte decimal. Content is 1-byte scale + 4-byte little-endian signed integer. + static constexpr int32_t kDecimal4 = 8; + /// 8-byte decimal. Content is 1-byte scale + 8-byte little-endian signed integer. + static constexpr int32_t kDecimal8 = 9; + /// 16-byte decimal. Content is 1-byte scale + 16-byte little-endian signed integer. + static constexpr int32_t kDecimal16 = 10; + /// Date value. Content is 4-byte little-endian signed integer that represents the number of + /// days from the Unix epoch. + static constexpr int32_t kDate = 11; + /// Timestamp value. Content is 8-byte little-endian signed integer that represents the number + /// of microseconds elapsed since the Unix epoch, 1970-01-01 00:00:00 UTC. It is displayed to + /// users in their local time zones and may be displayed differently depending on the + /// execution environment. + static constexpr int32_t kTimestamp = 12; + /// Timestamp_ntz value. It has the same content as `kTimestamp` but should always be + /// interpreted as if the local time zone is UTC. + static constexpr int32_t kTimestampNtz = 13; + /// 4-byte IEEE float. + static constexpr int32_t kFloat = 14; + /// Binary value. The content is (4-byte little-endian unsigned integer representing the + /// binary size) + (size bytes of binary content). + static constexpr int32_t kBinary = 15; + /// Long string value. The content is (4-byte little-endian unsigned integer representing the + /// string size) + (size bytes of string content). + static constexpr int32_t kLongStr = 16; + /// UUID, 16-byte big-endian. + static constexpr int32_t kUuid = 20; + + /// The only acceptable variant version. It is stored in the lower 4 bits of the first + /// metadata byte. + static constexpr uint8_t kVersion = 1; + static constexpr uint8_t kVersionMask = 0x0F; + + static constexpr int32_t kU8Max = 0xFF; + static constexpr int32_t kU16Max = 0xFFFF; + static constexpr int32_t kU24Max = 0xFFFFFF; + static constexpr int32_t kU24Size = 3; + static constexpr int32_t kU32Size = 4; + + /// Both variant value and variant metadata need to be no longer than 128MiB. + static constexpr int32_t kSizeLimit = 128 * 1024 * 1024; + + static constexpr int32_t kMaxDecimal4Precision = 9; + static constexpr int32_t kMaxDecimal8Precision = 18; + static constexpr int32_t kMaxDecimal16Precision = 38; + + /// Object field lookup switches from linear search to binary search when the object size + /// reaches this threshold. + static constexpr int32_t kBinarySearchThreshold = 32; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get.cpp b/src/paimon/common/data/variant/variant_get.cpp new file mode 100644 index 00000000..9bb8bab5 --- /dev/null +++ b/src/paimon/common/data/variant/variant_get.cpp @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_get.h" + +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_json_utils.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/casting/cast_executor_factory.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" + +namespace paimon { + +namespace { + +Result> InvalidCast(const std::shared_ptr& variant, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args) { + if (cast_args.fail_on_error) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return Status::Invalid(fmt::format("Invalid cast {} to {}", json, target_type->ToString())); + } + return std::optional(std::nullopt); +} + +Result> CastVariant(const std::shared_ptr& variant, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args) { + switch (target_type->id()) { + case arrow::Type::type::STRUCT: + case arrow::Type::type::LIST: + case arrow::Type::type::MAP: + return Status::NotImplemented(fmt::format( + "variant_get to nested type {} is not supported", target_type->ToString())); + default: + break; + } + + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant->GetType()); + if (variant_type == VariantValueType::kNull) { + return std::optional(std::nullopt); + } + + bool target_is_string = target_type->id() == arrow::Type::type::STRING; + if (variant_type == VariantValueType::kUuid) { + // There's no UUID type in Paimon. We only allow it to be cast to string. + if (target_is_string) { + PAIMON_ASSIGN_OR_RAISE(std::string_view uuid, variant->GetUuid()); + std::string uuid_str = VariantBinaryUtil::UuidToString(uuid); + return std::optional( + Literal(FieldType::STRING, uuid_str.data(), uuid_str.size())); + } + return InvalidCast(variant, target_type, cast_args); + } + + std::optional input; + std::shared_ptr input_type; + switch (variant_type) { + case VariantValueType::kObject: + case VariantValueType::kArray: { + if (target_is_string) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return std::optional(Literal(FieldType::STRING, json.data(), json.size())); + } + return InvalidCast(variant, target_type, cast_args); + } + case VariantValueType::kBoolean: { + PAIMON_ASSIGN_OR_RAISE(bool value, variant->GetBoolean()); + input = Literal(value); + input_type = arrow::boolean(); + break; + } + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant->GetLong()); + input = Literal(value); + input_type = arrow::int64(); + break; + } + case VariantValueType::kString: { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant->GetString()); + input = Literal(FieldType::STRING, value.data(), value.size()); + input_type = arrow::utf8(); + break; + } + case VariantValueType::kDouble: { + PAIMON_ASSIGN_OR_RAISE(double value, variant->GetDouble()); + if (target_is_string) { + // Match `GenericVariant::ToJson` and Java's `Double.toString` instead of the + // arrow cast formatting. + std::string str = VariantJsonUtils::JavaDoubleToString(value); + return std::optional(Literal(FieldType::STRING, str.data(), str.size())); + } + input = Literal(value); + input_type = arrow::float64(); + break; + } + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, variant->GetDecimal()); + if (value.scale < 0) { + // `paimon::Decimal` requires a non-negative scale; scale the value back up. + for (; value.scale < 0; ++value.scale) { + value.unscaled *= 10; + } + } + int32_t precision = std::max(value.Precision(), value.scale); + int32_t scale = value.scale; + input = Literal(Decimal(precision, scale, value.unscaled)); + input_type = arrow::decimal128(precision, scale); + break; + } + case VariantValueType::kDate: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant->GetLong()); + input = Literal(FieldType::DATE, static_cast(value)); + input_type = arrow::date32(); + break; + } + case VariantValueType::kFloat: { + PAIMON_ASSIGN_OR_RAISE(float value, variant->GetFloat()); + if (target_is_string) { + // Match `GenericVariant::ToJson` and Java's `Float.toString`. + std::string str = VariantJsonUtils::JavaFloatToString(value); + return std::optional(Literal(FieldType::STRING, str.data(), str.size())); + } + input = Literal(value); + input_type = arrow::float32(); + break; + } + case VariantValueType::kBinary: { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant->GetBinary()); + input = Literal(FieldType::BINARY, value.data(), value.size()); + input_type = arrow::binary(); + break; + } + case VariantValueType::kTimestamp: + case VariantValueType::kTimestampNtz: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, variant->GetLong()); + // Floor the division so negative epochs keep a non-negative sub-millisecond part. + int64_t millis = micros / 1000; + auto sub_micros = static_cast(micros % 1000); + if (sub_micros < 0) { + millis -= 1; + sub_micros += 1000; + } + input = Literal(Timestamp::FromEpochMillis(millis, sub_micros * 1000)); + input_type = variant_type == VariantValueType::kTimestamp + ? arrow::timestamp(arrow::TimeUnit::MICRO, "UTC") + : arrow::timestamp(arrow::TimeUnit::MICRO); + break; + } + default: + return Status::Invalid(fmt::format("Unsupported variant type in variant_get: {}", + static_cast(variant_type))); + } + + if (input_type->Equals(*target_type)) { + return input; + } + + PAIMON_ASSIGN_OR_RAISE(FieldType input_field_type, + FieldTypeUtils::ConvertToFieldType(input_type->id())); + PAIMON_ASSIGN_OR_RAISE(FieldType target_field_type, + FieldTypeUtils::ConvertToFieldType(target_type->id())); + std::shared_ptr executor = + CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(input_field_type, + target_field_type); + if (executor == nullptr) { + return InvalidCast(variant, target_type, cast_args); + } + Result cast_result = executor->Cast(*input, target_type); + if (!cast_result.ok()) { + return InvalidCast(variant, target_type, cast_args); + } + return std::optional(std::move(cast_result).value()); +} + +Status AppendLiteralToBuilder(const Literal& literal, + const std::shared_ptr& target_type, + arrow::ArrayBuilder* builder) { + switch (target_type->id()) { + case arrow::Type::type::BOOL: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT8: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT16: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT32: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT64: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::FLOAT: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::DOUBLE: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::STRING: + return ToPaimonStatus(static_cast(builder)->Append( + literal.GetValue())); + case arrow::Type::type::BINARY: + return ToPaimonStatus(static_cast(builder)->Append( + literal.GetValue())); + case arrow::Type::type::DATE32: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::TIMESTAMP: { + auto timestamp = literal.GetValue(); + const auto& timestamp_type = static_cast(*target_type); + int64_t value; + switch (timestamp_type.unit()) { + case arrow::TimeUnit::SECOND: + value = timestamp.GetMillisecond() / 1000; + break; + case arrow::TimeUnit::MILLI: + value = timestamp.GetMillisecond(); + break; + case arrow::TimeUnit::MICRO: + value = timestamp.ToMicrosecond(); + break; + case arrow::TimeUnit::NANO: + value = timestamp.ToNanosecond(); + break; + default: + return Status::Invalid("Unsupported timestamp unit"); + } + return ToPaimonStatus(static_cast(builder)->Append(value)); + } + case arrow::Type::type::DECIMAL128: { + auto decimal = literal.GetValue(); + arrow::Decimal128 value(static_cast(decimal.HighBits()), decimal.LowBits()); + return ToPaimonStatus(static_cast(builder)->Append(value)); + } + default: + return Status::Invalid( + fmt::format("Unsupported variant_get target type: {}", target_type->ToString())); + } +} + +} // namespace + +Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + arrow::ArrayBuilder* builder) { + if (variant == nullptr) { + return ToPaimonStatus(builder->AppendNull()); + } + + auto invalid_cast = [&]() -> Status { + if (cast_args.fail_on_error) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return Status::Invalid( + fmt::format("Invalid cast {} to {}", json, target_field->type()->ToString())); + } + return ToPaimonStatus(builder->AppendNull()); + }; + + if (VariantTypeUtils::IsVariantField(target_field)) { + VariantBuilder variant_builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(variant_builder.AppendVariant(*variant)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, variant_builder.Build(pool)); + auto* struct_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + PAIMON_ASSIGN_OR_RAISE(std::string_view value, copied->Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(struct_builder->field_builder(0))->Append(value)); + return ToPaimonStatus(static_cast(struct_builder->field_builder(1)) + ->Append(copied->Metadata())); + } + + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant->GetType()); + if (variant_type == VariantValueType::kNull) { + return ToPaimonStatus(builder->AppendNull()); + } + + switch (target_field->type()->id()) { + case arrow::Type::type::STRUCT: { + if (variant_type != VariantValueType::kObject) { + return invalid_cast(); + } + auto* struct_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + const auto& struct_type = static_cast(*target_field->type()); + for (int i = 0; i < struct_type.num_fields(); ++i) { + const std::shared_ptr& child_field = struct_type.field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + variant->GetFieldByKey(child_field->name())); + PAIMON_RETURN_NOT_OK(CastToBuilder(child, child_field, cast_args, pool, + struct_builder->field_builder(i))); + } + return Status::OK(); + } + case arrow::Type::type::MAP: { + const auto& map_type = static_cast(*target_field->type()); + if (map_type.key_type()->id() != arrow::Type::type::STRING || + variant_type != VariantValueType::kObject) { + return invalid_cast(); + } + auto* map_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Append()); + PAIMON_ASSIGN_OR_RAISE(int32_t object_size, variant->ObjectSize()); + for (int32_t i = 0; i < object_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant->GetFieldAtIndex(i)); + if (!field.has_value()) { + return Status::Invalid(fmt::format("Malformed variant object at index {}", i)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(map_builder->key_builder()) + ->Append(field->key)); + PAIMON_RETURN_NOT_OK(CastToBuilder(field->value, map_type.item_field(), cast_args, + pool, map_builder->item_builder())); + } + return Status::OK(); + } + case arrow::Type::type::LIST: { + if (variant_type != VariantValueType::kArray) { + return invalid_cast(); + } + auto* list_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Append()); + const auto& list_type = static_cast(*target_field->type()); + PAIMON_ASSIGN_OR_RAISE(int32_t array_size, variant->ArraySize()); + for (int32_t i = 0; i < array_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant->GetElementAtIndex(i)); + PAIMON_RETURN_NOT_OK(CastToBuilder(element, list_type.value_field(), cast_args, + pool, list_builder->value_builder())); + } + return Status::OK(); + } + default: { + PAIMON_ASSIGN_OR_RAISE(std::optional literal, + CastVariant(variant, target_field->type(), cast_args)); + if (!literal.has_value()) { + return ToPaimonStatus(builder->AppendNull()); + } + return AppendLiteralToBuilder(*literal, target_field->type(), builder); + } + } +} + +Result> VariantGetExecutor::GetAsArrow( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_field, const VariantCastArgs& cast_args, + const std::shared_ptr& pool, const std::shared_ptr& arrow_pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, ExtractByPath(variant, path)); + std::unique_ptr builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::MakeBuilder(arrow_pool.get(), target_field->type(), &builder)); + PAIMON_RETURN_NOT_OK(CastToBuilder(extracted, target_field, cast_args, pool, builder.get())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + return array; +} + +Result> VariantGetExecutor::ExtractByPath( + const std::shared_ptr& variant, const std::string& path) { + PAIMON_ASSIGN_OR_RAISE(std::vector segments, + VariantPathSegment::Parse(path)); + std::shared_ptr current = variant; + for (const VariantPathSegment& segment : segments) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, current->GetType()); + if (segment.kind == VariantPathSegment::Kind::kObjectExtraction && + type == VariantValueType::kObject) { + PAIMON_ASSIGN_OR_RAISE(current, current->GetFieldByKey(segment.key)); + } else if (segment.kind == VariantPathSegment::Kind::kArrayExtraction && + type == VariantValueType::kArray) { + PAIMON_ASSIGN_OR_RAISE(current, current->GetElementAtIndex(segment.index)); + } else { + return std::shared_ptr(nullptr); + } + if (current == nullptr) { + return std::shared_ptr(nullptr); + } + } + return current; +} + +Result> VariantGetExecutor::Get( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_type, const VariantCastArgs& cast_args) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, ExtractByPath(variant, path)); + if (extracted == nullptr) { + return std::optional(std::nullopt); + } + return CastVariant(extracted, target_type, cast_args); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get.h b/src/paimon/common/data/variant/variant_get.h new file mode 100644 index 00000000..9cfa8d33 --- /dev/null +++ b/src/paimon/common/data/variant/variant_get.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/data/variant.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class ArrayBuilder; +class DataType; +class Field; +class MemoryPool; +} // namespace arrow + +namespace paimon { + +/// Implements `variant_get` semantics: extracting a sub-variant by a JSONPath-like path and +/// casting it to a target type. +class VariantGetExecutor { + public: + VariantGetExecutor() = delete; + ~VariantGetExecutor() = delete; + + /// Extracts a sub-variant value according to a path which starts with a `$`, e.g. `$.key`, + /// `$['key']`, `$["key"]`, `$.array[0]`. Returns nullptr if the path does not match the + /// variant structure. + static Result> ExtractByPath( + const std::shared_ptr& variant, const std::string& path); + + /// Extracts a sub-variant by `path` and casts it to `target_type`. Returns nullopt for SQL + /// NULL (unmatched path, variant null, or an invalid cast with `fail_on_error == false`). + /// + /// Nested target types (ROW/ARRAY/MAP/VARIANT) are not supported by the `Literal` result + /// type; use `CastToBuilder` / `GetAsArrow` for structured results. + static Result> Get(const std::shared_ptr& variant, + const std::string& path, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args); + + /// Casts `variant` to the type of `target_field` and appends the result to `builder`. + /// + /// Supported targets beyond scalars: a variant-marked struct field (the variant is deeply + /// re-encoded), STRUCT (from a variant object, children matched by name; unmatched children + /// are null), MAP with string keys (from a variant object), and LIST (from a variant array). + /// A nullptr `variant`, a variant null, or an invalid cast with `fail_on_error == false` + /// appends null. + static Status CastToBuilder(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + arrow::ArrayBuilder* builder); + + /// Extracts a sub-variant by `path`, casts it to the (possibly nested) type of + /// `target_field`, and returns a length-1 arrow array holding the result (a null slot + /// represents SQL NULL). `arrow_pool` must outlive the returned array. + static Result> GetAsArrow( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_field, const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + const std::shared_ptr& arrow_pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get_test.cpp b/src/paimon/common/data/variant/variant_get_test.cpp new file mode 100644 index 00000000..159a573d --- /dev/null +++ b/src/paimon/common/data/variant/variant_get_test.cpp @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_get.h" + +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantGetTest : public ::testing::Test { + public: + void SetUp() override { + // The same document as the Java GenericVariantTest#testVariantGet. + std::string json = + "{\n" + " \"object\": {\n" + " \"name\": \"Apache Paimon\",\n" + " \"age\": 2,\n" + " \"address\": {\n" + " \"street\": \"Main St\",\n" + " \"city\": \"Hangzhou\"\n" + " }\n" + " },\n" + " \"array\": [1, 2, 3, 4, 5],\n" + " \"string\": \"Hello, World!\",\n" + " \"long\": 12345678901234,\n" + " \"double\": 1.0123456789012345678901234567890123456789,\n" + " \"decimal\": 100.99,\n" + " \"boolean1\": true,\n" + " \"boolean2\": false,\n" + " \"nullField\": null\n" + "}\n"; + ASSERT_OK_AND_ASSIGN(variant_, GenericVariant::FromJson(json, pool_)); + cast_args_.fail_on_error = false; + } + + std::optional Get(const std::string& path, + const std::shared_ptr& target) { + auto result = VariantGetExecutor::Get(variant_, path, target, cast_args_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + std::string GetString(const std::string& path, const std::shared_ptr& target) { + std::optional literal = Get(path, target); + EXPECT_TRUE(literal.has_value()); + return literal->ToString(); + } + + Result> GetAsArrow( + const std::string& path, const std::shared_ptr& target_field) { + return VariantGetExecutor::GetAsArrow(variant_, path, target_field, cast_args_, pool_, + arrow_pool_); + } + + // Builds a single-scalar variant using the direct append API, which can encode types that + // JSON parsing never produces (uuid/date/float/binary/timestamp). + std::shared_ptr BuildScalar( + const std::function& append) { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + Status st = append(builder); + EXPECT_TRUE(st.ok()) << st.ToString(); + auto result = builder.Build(pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr arrow_pool_ = GetArrowPool(pool_); + std::shared_ptr variant_; + VariantCastArgs cast_args_; +}; + +TEST_F(VariantGetTest, PathSegmentParse) { + ASSERT_OK_AND_ASSIGN(auto segments, + VariantPathSegment::Parse("$[\"object\"]['address'].city[3]")); + ASSERT_EQ(segments.size(), 4); + ASSERT_EQ(segments[0].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[0].key, "object"); + ASSERT_EQ(segments[1].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[1].key, "address"); + ASSERT_EQ(segments[2].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[2].key, "city"); + ASSERT_EQ(segments[3].kind, VariantPathSegment::Kind::kArrayExtraction); + ASSERT_EQ(segments[3].index, 3); + + ASSERT_OK_AND_ASSIGN(auto root_only, VariantPathSegment::Parse("$")); + ASSERT_TRUE(root_only.empty()); + + // Java parity: the root `$` is located anywhere in the path (Java uses Matcher#find), so a + // prefix before the first `$` is tolerated and ignored. + ASSERT_OK_AND_ASSIGN(auto prefixed, VariantPathSegment::Parse("abc$.x")); + ASSERT_EQ(prefixed.size(), 1); + ASSERT_EQ(prefixed[0].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(prefixed[0].key, "x"); + + ASSERT_NOK(VariantPathSegment::Parse("")); + ASSERT_NOK(VariantPathSegment::Parse("no_root")); + ASSERT_NOK(VariantPathSegment::Parse("$.")); + ASSERT_NOK(VariantPathSegment::Parse("$[abc]")); + ASSERT_NOK(VariantPathSegment::Parse("$['unterminated]")); +} + +TEST_F(VariantGetTest, ScalarTargets) { + ASSERT_EQ(GetString("$.string", arrow::utf8()), "Hello, World!"); + auto long_value = Get("$.long", arrow::int64()); + ASSERT_TRUE(long_value.has_value()); + ASSERT_EQ(long_value->GetValue(), 12345678901234LL); + ASSERT_EQ(GetString("$.long", arrow::utf8()), "12345678901234"); + auto double_value = Get("$.double", arrow::float64()); + ASSERT_TRUE(double_value.has_value()); + ASSERT_DOUBLE_EQ(double_value->GetValue(), 1.0123456789012346); + auto decimal_value = Get("$.decimal", arrow::decimal128(5, 2)); + ASSERT_TRUE(decimal_value.has_value()); + ASSERT_EQ(decimal_value->GetValue().ToUnscaledLong(), 10099); + ASSERT_EQ(GetString("$.decimal", arrow::utf8()), "100.99"); + auto bool1 = Get("$.boolean1", arrow::boolean()); + ASSERT_TRUE(bool1.has_value()); + ASSERT_TRUE(bool1->GetValue()); + auto bool2 = Get("$.boolean2", arrow::boolean()); + ASSERT_TRUE(bool2.has_value()); + ASSERT_FALSE(bool2->GetValue()); + // Variant null maps to SQL NULL. + ASSERT_FALSE(Get("$.nullField", arrow::boolean()).has_value()); + auto elem = Get("$.array[3]", arrow::int64()); + ASSERT_TRUE(elem.has_value()); + ASSERT_EQ(elem->GetValue(), 4); +} + +TEST_F(VariantGetTest, ContainerToJsonString) { + ASSERT_EQ(GetString("$.object", arrow::utf8()), + "{\"address\":{\"city\":\"Hangzhou\",\"street\":\"Main St\"},\"age\":2,\"name\":" + "\"Apache Paimon\"}"); + ASSERT_EQ(GetString("$.object.name", arrow::utf8()), "Apache Paimon"); + ASSERT_EQ(GetString("$.object.address.street", arrow::utf8()), "Main St"); + ASSERT_EQ(GetString("$[\"object\"]['address'].city", arrow::utf8()), "Hangzhou"); + ASSERT_EQ(GetString("$.array", arrow::utf8()), "[1,2,3,4,5]"); +} + +TEST_F(VariantGetTest, UnmatchedPathAndInvalidCast) { + // A path that does not exist yields SQL NULL. + ASSERT_FALSE(Get("$.missing", arrow::utf8()).has_value()); + ASSERT_FALSE(Get("$.string[0]", arrow::utf8()).has_value()); + ASSERT_FALSE(Get("$.array.key", arrow::utf8()).has_value()); + // An invalid cast yields SQL NULL when fail_on_error is false. + ASSERT_FALSE(Get("$.object", arrow::int64()).has_value()); + // ... and an error when fail_on_error is true. + cast_args_.fail_on_error = true; + auto result = VariantGetExecutor::Get(variant_, "$.object", arrow::int64(), cast_args_); + ASSERT_NOK(result); +} + +TEST_F(VariantGetTest, NestedTargetsNotImplemented) { + auto result = + VariantGetExecutor::Get(variant_, "$.array", arrow::list(arrow::int32()), cast_args_); + ASSERT_TRUE(result.status().IsNotImplemented()); +} + +TEST_F(VariantGetTest, BinaryAndTimestampSources) { + // Java-encoded data can carry binary/timestamp scalars that JSON parsing never produces. + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendBinary(std::string_view("\x01\x02\x03", 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::binary(), cast_args_)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue(), std::string("\x01\x02\x03", 3)); + } + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendTimestamp(1700000000123456)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN( + std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + cast_args_)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue().ToMicrosecond(), 1700000000123456); + } + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendTimestampNtz(-1001)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN( + std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::timestamp(arrow::TimeUnit::MICRO), + cast_args_)); + ASSERT_TRUE(literal.has_value()); + // Negative epochs must floor, not truncate toward zero. + ASSERT_EQ(literal->GetValue().ToMicrosecond(), -1001); + } +} + +TEST_F(VariantGetTest, ExtractByPath) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr address, + VariantGetExecutor::ExtractByPath(variant_, "$.object.address")); + ASSERT_NE(address, nullptr); + ASSERT_OK_AND_ASSIGN(std::string json, address->ToJson()); + ASSERT_EQ(json, "{\"city\":\"Hangzhou\",\"street\":\"Main St\"}"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, + VariantGetExecutor::ExtractByPath(variant_, "$.object.missing")); + ASSERT_EQ(missing, nullptr); +} + +TEST_F(VariantGetTest, CastToStructTarget) { + auto target = arrow::field( + "r", arrow::struct_( + {arrow::field("name", arrow::utf8()), arrow::field("age", arrow::int64()), + arrow::field("address", arrow::struct_({arrow::field("city", arrow::utf8())})), + arrow::field("missing", arrow::utf8())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.object", target)); + ASSERT_EQ(array->length(), 1); + const auto& row = static_cast(*array); + ASSERT_FALSE(row.IsNull(0)); + ASSERT_EQ(static_cast(*row.field(0)).GetString(0), "Apache Paimon"); + ASSERT_EQ(static_cast(*row.field(1)).Value(0), 2); + const auto& address = static_cast(*row.field(2)); + ASSERT_EQ(static_cast(*address.field(0)).GetString(0), "Hangzhou"); + // A target field absent from the variant object is null. + ASSERT_TRUE(row.field(3)->IsNull(0)); + + // With fail_on_error == false a child that cannot cast becomes null while the parent row + // stays non-null. + auto mixed_target = arrow::field("r", arrow::struct_({arrow::field("string", arrow::int64()), + arrow::field("long", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$", mixed_target)); + const auto& mixed_row = static_cast(*array); + ASSERT_FALSE(mixed_row.IsNull(0)); + // "string" holds "Hello, World!", which cannot cast to int64. + ASSERT_TRUE(mixed_row.field(0)->IsNull(0)); + ASSERT_FALSE(mixed_row.field(1)->IsNull(0)); + ASSERT_EQ(static_cast(*mixed_row.field(1)).Value(0), 12345678901234); +} + +TEST_F(VariantGetTest, CastToStructFromNonObject) { + auto target = arrow::field("r", arrow::struct_({arrow::field("a", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.array", target)); + ASSERT_TRUE(array->IsNull(0)); + cast_args_.fail_on_error = true; + auto result = + VariantGetExecutor::GetAsArrow(variant_, "$.array", target, cast_args_, pool_, arrow_pool_); + ASSERT_NOK(result); +} + +TEST_F(VariantGetTest, CastToMapTarget) { + auto target = arrow::field("m", arrow::map(arrow::utf8(), arrow::utf8())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + GetAsArrow("$.object.address", target)); + const auto& map = static_cast(*array); + ASSERT_EQ(map.value_length(0), 2); + const auto& keys = static_cast(*map.keys()); + const auto& items = static_cast(*map.items()); + ASSERT_EQ(keys.GetString(0), "city"); + ASSERT_EQ(items.GetString(0), "Hangzhou"); + ASSERT_EQ(keys.GetString(1), "street"); + ASSERT_EQ(items.GetString(1), "Main St"); + // A map with a non-string key type is an invalid cast. + auto bad_target = arrow::field("m", arrow::map(arrow::int64(), arrow::utf8())); + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.object.address", bad_target)); + ASSERT_TRUE(array->IsNull(0)); +} + +TEST_F(VariantGetTest, CastToListTarget) { + auto target = arrow::field("l", arrow::list(arrow::int64())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.array", target)); + const auto& list = static_cast(*array); + ASSERT_EQ(list.value_length(0), 5); + const auto& values = static_cast(*list.values()); + for (int64_t i = 0; i < 5; ++i) { + ASSERT_EQ(values.Value(i), i + 1); + } + auto list_of_structs = + arrow::field("l", arrow::list(arrow::struct_({arrow::field("a", arrow::int64())}))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, + GenericVariant::FromJson("[{\"a\": 1}, {\"a\": 2}]", pool_)); + ASSERT_OK_AND_ASSIGN(array, VariantGetExecutor::GetAsArrow(rows, "$", list_of_structs, + cast_args_, pool_, arrow_pool_)); + const auto& struct_list = static_cast(*array); + ASSERT_EQ(struct_list.value_length(0), 2); + const auto& elements = static_cast(*struct_list.values()); + ASSERT_EQ(static_cast(*elements.field(0)).Value(0), 1); + ASSERT_EQ(static_cast(*elements.field(0)).Value(1), 2); +} + +TEST_F(VariantGetTest, CastToVariantTarget) { + // Re-encodes the extracted sub-variant against a fresh metadata dictionary. + auto target = VariantTypeUtils::ToArrowField("v", /*nullable=*/true, {}); + auto read_variant_json = [&](const arrow::Array& array) { + const auto& row = static_cast(array); + EXPECT_FALSE(row.IsNull(0)); + std::string_view value = static_cast(*row.field(0)).GetView(0); + std::string_view metadata = + static_cast(*row.field(1)).GetView(0); + EXPECT_OK_AND_ASSIGN(std::shared_ptr copied, + GenericVariant::Create(value, metadata, pool_)); + EXPECT_OK_AND_ASSIGN(std::string json, copied->ToJson()); + return json; + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + GetAsArrow("$.object.address", target)); + ASSERT_EQ(read_variant_json(*array), "{\"city\":\"Hangzhou\",\"street\":\"Main St\"}"); + + // A variant null cast to a VARIANT target stays an encoded variant null (a non-null row + // whose value renders as JSON null); only scalar targets turn a variant null into SQL NULL. + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.nullField", target)); + ASSERT_EQ(read_variant_json(*array), "null"); + // An unmatched path is SQL NULL by contrast. + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.missing", target)); + ASSERT_TRUE(array->IsNull(0)); +} + +TEST_F(VariantGetTest, NestedTargetNullSemantics) { + auto target = arrow::field("r", arrow::struct_({arrow::field("a", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.missing", target)); + ASSERT_TRUE(array->IsNull(0)); + // A variant null yields SQL NULL even with fail_on_error == true. + cast_args_.fail_on_error = true; + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.nullField", target)); + ASSERT_TRUE(array->IsNull(0)); +} + +TEST_F(VariantGetTest, UuidSourceCastsToStringOnly) { + std::string uuid_bytes(16, '\0'); + for (int i = 0; i < 16; ++i) { + uuid_bytes[i] = static_cast(i); + } + std::shared_ptr variant = + BuildScalar([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); }); + // A UUID has no Paimon type, so it can only be rendered as its canonical string. + ASSERT_OK_AND_ASSIGN(std::optional as_string, + VariantGetExecutor::Get(variant, "$", arrow::utf8(), cast_args_)); + ASSERT_TRUE(as_string.has_value()); + ASSERT_EQ(as_string->GetValue(), "00010203-0405-0607-0809-0a0b0c0d0e0f"); + // Any non-string target is an invalid cast, which is SQL NULL when fail_on_error is false. + ASSERT_OK_AND_ASSIGN(std::optional as_long, + VariantGetExecutor::Get(variant, "$", arrow::int64(), cast_args_)); + ASSERT_FALSE(as_long.has_value()); + cast_args_.fail_on_error = true; + ASSERT_NOK(VariantGetExecutor::Get(variant, "$", arrow::int64(), cast_args_)); +} + +TEST_F(VariantGetTest, FloatingPointToStringMatchesJava) { + // Doubles and floats are stringified via the Java formatting, not the arrow cast. + ASSERT_EQ(GetString("$.double", arrow::utf8()), "1.0123456789012346"); + std::shared_ptr f = + BuildScalar([](VariantBuilder& b) { return b.AppendFloat(1.5f); }); + ASSERT_OK_AND_ASSIGN(std::optional as_string, + VariantGetExecutor::Get(f, "$", arrow::utf8(), cast_args_)); + ASSERT_TRUE(as_string.has_value()); + ASSERT_EQ(as_string->GetValue(), "1.5"); + ASSERT_OK_AND_ASSIGN(std::optional as_float, + VariantGetExecutor::Get(f, "$", arrow::float32(), cast_args_)); + ASSERT_TRUE(as_float.has_value()); + ASSERT_EQ(as_float->GetValue(), 1.5f); +} + +TEST_F(VariantGetTest, DateSource) { + std::shared_ptr d = + BuildScalar([](VariantBuilder& b) { return b.AppendDate(19000); }); + ASSERT_OK_AND_ASSIGN(std::optional as_date, + VariantGetExecutor::Get(d, "$", arrow::date32(), cast_args_)); + ASSERT_TRUE(as_date.has_value()); + ASSERT_EQ(as_date->GetValue(), 19000); +} + +TEST_F(VariantGetTest, MissingCastExecutorYieldsNull) { + // No binary->int64 cast executor exists, so the cast is treated as invalid (SQL NULL). + std::shared_ptr bin = BuildScalar( + [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02", 2)); }); + ASSERT_OK_AND_ASSIGN(std::optional as_long, + VariantGetExecutor::Get(bin, "$", arrow::int64(), cast_args_)); + ASSERT_FALSE(as_long.has_value()); +} + +TEST_F(VariantGetTest, ScalarArrowBuilders) { + auto build_array = [&](const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& type) { + auto result = VariantGetExecutor::GetAsArrow(variant, path, arrow::field("x", type), + cast_args_, pool_, arrow_pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + }; + + std::shared_ptr boolean = build_array(variant_, "$.boolean1", arrow::boolean()); + ASSERT_TRUE(static_cast(*boolean).Value(0)); + + // INT8/INT16/INT32 each require a narrowing cast from the int64 source. + std::shared_ptr int8 = build_array(variant_, "$.object.age", arrow::int8()); + ASSERT_EQ(static_cast(*int8).Value(0), 2); + std::shared_ptr int16 = build_array(variant_, "$.object.age", arrow::int16()); + ASSERT_EQ(static_cast(*int16).Value(0), 2); + std::shared_ptr int32 = build_array(variant_, "$.object.age", arrow::int32()); + ASSERT_EQ(static_cast(*int32).Value(0), 2); + + std::shared_ptr float64 = build_array(variant_, "$.double", arrow::float64()); + ASSERT_DOUBLE_EQ(static_cast(*float64).Value(0), 1.0123456789012346); + + std::shared_ptr f = + BuildScalar([](VariantBuilder& b) { return b.AppendFloat(1.5f); }); + std::shared_ptr float32 = build_array(f, "$", arrow::float32()); + ASSERT_EQ(static_cast(*float32).Value(0), 1.5f); + + std::shared_ptr decimal = + build_array(variant_, "$.decimal", arrow::decimal128(5, 2)); + ASSERT_EQ(static_cast(*decimal).FormatValue(0), "100.99"); + + std::shared_ptr bin = BuildScalar( + [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02\x03", 3)); }); + std::shared_ptr binary = build_array(bin, "$", arrow::binary()); + ASSERT_EQ(static_cast(*binary).GetView(0), + std::string_view("\x01\x02\x03", 3)); + + std::shared_ptr d = + BuildScalar([](VariantBuilder& b) { return b.AppendDate(19000); }); + std::shared_ptr date = build_array(d, "$", arrow::date32()); + ASSERT_EQ(static_cast(*date).Value(0), 19000); + + // Only the microsecond unit matches the variant's internal timestamp representation; other + // units require a timestamp->timestamp literal cast that is intentionally unsupported. + std::shared_ptr ts = + BuildScalar([](VariantBuilder& b) { return b.AppendTimestamp(1700000000123456); }); + std::shared_ptr timestamp = + build_array(ts, "$", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")); + ASSERT_EQ(static_cast(*timestamp).Value(0), 1700000000123456); +} + +TEST_F(VariantGetTest, ListFromNonArrayIsNull) { + auto target = arrow::field("l", arrow::list(arrow::int64())); + // An object cannot cast to a list: SQL NULL when fail_on_error is false ... + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.object", target)); + ASSERT_TRUE(array->IsNull(0)); + // ... and an error when fail_on_error is true. + cast_args_.fail_on_error = true; + auto result = VariantGetExecutor::GetAsArrow(variant_, "$.object", target, cast_args_, pool_, + arrow_pool_); + ASSERT_NOK(result); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_json_utils.cpp b/src/paimon/common/data/variant/variant_json_utils.cpp new file mode 100644 index 00000000..897c000e --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils.cpp @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_json_utils.h" + +#include +#include +#include +#include + +#include "arrow/vendored/datetime.h" +#include "fmt/format.h" + +namespace paimon { + +namespace { + +constexpr int64_t kMicrosPerSecond = 1000000LL; +constexpr int64_t kMicrosPerDay = 86400LL * kMicrosPerSecond; + +// Formats the shortest round-trip decimal digits + decimal exponent into the textual form of +// `java.lang.Double.toString` / `java.lang.Float.toString`: plain notation when +// `1e-3 <= |value| < 1e7`, computerized scientific notation otherwise. +std::string JavaFloatingToString(std::string_view digits, int32_t exp, bool negative) { + std::string result; + if (negative) { + result.push_back('-'); + } + if (exp >= -3 && exp < 7) { + if (exp >= 0) { + size_t int_digits = static_cast(exp) + 1; + if (digits.size() > int_digits) { + result.append(digits, 0, int_digits); + result.push_back('.'); + result.append(digits.substr(int_digits)); + } else { + result.append(digits); + result.append(int_digits - digits.size(), '0'); + result.append(".0"); + } + } else { + result.append("0."); + result.append(static_cast(-exp) - 1, '0'); + result.append(digits); + } + } else { + result.push_back(digits[0]); + result.push_back('.'); + if (digits.size() > 1) { + result.append(digits.substr(1)); + } else { + result.push_back('0'); + } + result.push_back('E'); + result.append(std::to_string(exp)); + } + return result; +} + +template +std::string FloatingToJavaString(T value) { + if (std::isnan(value)) { + return "NaN"; + } + if (std::isinf(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + if (value == 0) { + return std::signbit(value) ? "-0.0" : "0.0"; + } + // The shortest round-trip representation in scientific form, e.g. `1.234e+05`. gcc 8's + // lacks floating-point to_chars, so probe increasing precision until the value + // round-trips; dropping a trailing zero digit is an exact rounding, so the first precision + // that round-trips carries no trailing zeros. + constexpr int kMaxFractionDigits = std::is_same_v ? 8 : 16; + char buf[64]; + int len = 0; + for (int precision = 0; precision <= kMaxFractionDigits; ++precision) { + len = std::snprintf(buf, sizeof(buf), "%.*e", precision, static_cast(value)); + T parsed; + if constexpr (std::is_same_v) { + parsed = std::strtof(buf, nullptr); + } else { + parsed = std::strtod(buf, nullptr); + } + if (parsed == value) { + break; + } + } + auto parse = [](std::string_view repr, std::string* digits, int32_t* exp, bool* negative) { + *negative = repr[0] == '-'; + if (*negative) { + repr.remove_prefix(1); + } + size_t e_pos = repr.find('e'); + std::string_view mantissa = repr.substr(0, e_pos); + size_t exp_start = e_pos + 1; + // `std::from_chars` does not accept a leading '+'. + if (repr[exp_start] == '+') { + ++exp_start; + } + *exp = 0; + std::from_chars(repr.data() + exp_start, repr.data() + repr.size(), *exp); + digits->clear(); + digits->push_back(mantissa[0]); + if (mantissa.size() > 2) { + digits->append(mantissa.substr(2)); + } + }; + std::string digits; + int32_t exp = 0; + bool negative = false; + parse(std::string_view(buf, static_cast(len)), &digits, &exp, &negative); + if (digits.size() == 1 && (exp < -3 || exp >= 8)) { + // Java's FloatingDecimal emits at least two significant digits when the first digit + // alone would terminate in scientific form; re-render the correctly rounded two-digit + // representation (e.g. `Double.MIN_VALUE` is `4.9E-324`, not `5.0E-324`). + len = std::snprintf(buf, sizeof(buf), "%.1e", static_cast(value)); + parse(std::string_view(buf, static_cast(len)), &digits, &exp, &negative); + } + return JavaFloatingToString(digits, exp, negative); +} + +// Appends a year like `java.time.LocalDate.toString`: absolute value padded to at least 4 +// digits; years above 9999 get a `+` prefix. +void AppendJavaYear(int64_t year, std::string* out) { + if (year < 0) { + out->push_back('-'); + year = -year; + out->append(fmt::format("{:04}", year)); + } else { + if (year > 9999) { + out->push_back('+'); + } + out->append(fmt::format("{:04}", year)); + } +} + +void AppendDate(int64_t days_since_epoch, std::string* out) { + using arrow_vendored::date::days; + using arrow_vendored::date::sys_days; + using arrow_vendored::date::year_month_day; + year_month_day ymd{sys_days{days{days_since_epoch}}}; + AppendJavaYear(static_cast(ymd.year()), out); + out->append(fmt::format("-{:02}-{:02}", static_cast(ymd.month()), + static_cast(ymd.day()))); +} + +int64_t FloorDiv(int64_t x, int64_t y) { + int64_t quotient = x / y; + if ((x % y != 0) && ((x < 0) != (y < 0))) { + --quotient; + } + return quotient; +} + +} // namespace + +void VariantJsonUtils::AppendEscapedJson(std::string_view str, std::string* out) { + out->push_back('"'); + for (char c : str) { + switch (c) { + case '"': + out->append("\\\""); + break; + case '\\': + out->append("\\\\"); + break; + case '\b': + out->append("\\b"); + break; + case '\f': + out->append("\\f"); + break; + case '\n': + out->append("\\n"); + break; + case '\r': + out->append("\\r"); + break; + case '\t': + out->append("\\t"); + break; + default: + if (static_cast(c) < 0x20) { + out->append(fmt::format("\\u{:04x}", static_cast(c))); + } else { + out->push_back(c); + } + } + } + out->push_back('"'); +} + +std::string VariantJsonUtils::JavaDoubleToString(double value) { + return FloatingToJavaString(value); +} + +std::string VariantJsonUtils::JavaFloatToString(float value) { + return FloatingToJavaString(value); +} + +std::string VariantJsonUtils::DateToString(int32_t days_since_epoch) { + std::string result; + AppendDate(days_since_epoch, &result); + return result; +} + +std::string VariantJsonUtils::TimestampToString(int64_t micros_since_epoch, int32_t offset_seconds, + bool with_offset) { + int64_t local_micros = micros_since_epoch + static_cast(offset_seconds) * 1000000; + int64_t days = FloorDiv(local_micros, kMicrosPerDay); + int64_t micros_of_day = local_micros - days * kMicrosPerDay; + std::string result; + AppendDate(days, &result); + int64_t seconds_of_day = micros_of_day / kMicrosPerSecond; + int64_t micros_of_second = micros_of_day % kMicrosPerSecond; + result.append(fmt::format(" {:02}:{:02}:{:02}", seconds_of_day / 3600, + (seconds_of_day / 60) % 60, seconds_of_day % 60)); + if (micros_of_second != 0) { + std::string fraction = fmt::format("{:06}", micros_of_second); + while (fraction.back() == '0') { + fraction.pop_back(); + } + result.push_back('.'); + result.append(fraction); + } + if (with_offset) { + int32_t abs_offset = offset_seconds >= 0 ? offset_seconds : -offset_seconds; + result.append(fmt::format("{}{:02}:{:02}", offset_seconds >= 0 ? '+' : '-', + abs_offset / 3600, (abs_offset / 60) % 60)); + } + return result; +} + +Result VariantJsonUtils::GetZoneOffsetSeconds(const std::string& zone_id, + int64_t micros_since_epoch) { + std::string_view id = zone_id; + if (id == "Z" || id == "UTC" || id == "GMT" || id == "UT") { + return 0; + } + // `UTC+08:00` style ids: strip the prefix and parse the remaining fixed offset. + if (id.size() > 3 && (id.substr(0, 3) == "UTC" || id.substr(0, 3) == "GMT")) { + id.remove_prefix(3); + } else if (id.size() > 2 && id.substr(0, 2) == "UT" && (id[2] == '+' || id[2] == '-')) { + id.remove_prefix(2); + } + if (!id.empty() && (id[0] == '+' || id[0] == '-')) { + bool negative = id[0] == '-'; + id.remove_prefix(1); + // Accepted forms: H, HH, HH:MM, HHMM, HH:MM:SS, HHMMSS. + std::string digits; + for (char c : id) { + if (c >= '0' && c <= '9') { + digits.push_back(c); + } else if (c != ':') { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + } + int32_t hours = 0; + int32_t minutes = 0; + int32_t seconds = 0; + if (digits.size() == 1 || digits.size() == 2) { + hours = std::stoi(digits); + } else if (digits.size() == 4) { + hours = std::stoi(digits.substr(0, 2)); + minutes = std::stoi(digits.substr(2, 2)); + } else if (digits.size() == 6) { + hours = std::stoi(digits.substr(0, 2)); + minutes = std::stoi(digits.substr(2, 2)); + seconds = std::stoi(digits.substr(4, 2)); + } else { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + if (hours > 18 || minutes > 59 || seconds > 59) { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + int32_t total = hours * 3600 + minutes * 60 + seconds; + return negative ? -total : total; + } + // IANA region id, resolved at the given instant (honoring DST). + try { + const auto* zone = arrow_vendored::date::locate_zone(zone_id); + std::chrono::time_point tp{ + std::chrono::microseconds(micros_since_epoch)}; + auto info = zone->get_info(tp); + return static_cast(info.offset.count()); + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("Invalid zone id: {}, {}", zone_id, e.what())); + } +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_json_utils.h b/src/paimon/common/data/variant/variant_json_utils.h new file mode 100644 index 00000000..bf40a3e2 --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// Helpers for rendering variant values as JSON text, matching the output of the Java +/// implementation (`GenericVariant.toJsonImpl`) character-for-character. +class VariantJsonUtils { + public: + VariantJsonUtils() = delete; + ~VariantJsonUtils() = delete; + + /// Appends `str` as a quoted JSON string with escaping into `out`. + static void AppendEscapedJson(std::string_view str, std::string* out); + + /// Formats a double like `java.lang.Double.toString`, e.g. `1.0`, `-0.001`, `1.0E7`. + static std::string JavaDoubleToString(double value); + + /// Formats a float like `java.lang.Float.toString`. + static std::string JavaFloatToString(float value); + + /// Formats days-since-epoch like `java.time.LocalDate.toString`, e.g. `2024-01-15`. + static std::string DateToString(int32_t days_since_epoch); + + /// Formats microseconds-since-epoch at the given offset like the Java formatter + /// `yyyy-MM-dd HH:mm:ss[.fraction]` (fraction with trailing zeros trimmed), optionally + /// followed by a `+HH:MM` offset suffix. + static std::string TimestampToString(int64_t micros_since_epoch, int32_t offset_seconds, + bool with_offset); + + /// Resolves the UTC offset (in seconds) of `zone_id` at the given instant. Supports fixed + /// offsets (`+08:00`, `-05:30`, optionally prefixed with `UTC`/`GMT`), `Z`, `UTC`, `GMT`, + /// and IANA region ids such as `Asia/Shanghai`. + static Result GetZoneOffsetSeconds(const std::string& zone_id, + int64_t micros_since_epoch); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_json_utils_test.cpp b/src/paimon/common/data/variant/variant_json_utils_test.cpp new file mode 100644 index 00000000..d2cf513e --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils_test.cpp @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_json_utils.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(VariantJsonUtilsTest, JavaDoubleToString) { + // Mirrors java.lang.Double#toString exactly. + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.0), "0.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-0.0), "-0.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::infinity()), + "Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-std::numeric_limits::infinity()), + "-Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::nan("")), "NaN"); + // Double.MIN_VALUE (the smallest positive subnormal) and Double.MAX_VALUE. + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::denorm_min()), + "4.9E-324"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::max()), + "1.7976931348623157E308"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::min()), + "2.2250738585072014E-308"); + + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0), "1.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-1.0), "-1.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-0.001), "-0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1e7), "1.0E7"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1234567.0), "1234567.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.001), "0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0E-3), "0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0E-4), "1.0E-4"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.1 + 0.2), "0.30000000000000004"); +} + +TEST(VariantJsonUtilsTest, JavaFloatToString) { + // Mirrors java.lang.Float#toString exactly. + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(0.0F), "0.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(-0.0F), "-0.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::infinity()), + "Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(-std::numeric_limits::infinity()), + "-Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::nanf("")), "NaN"); + // Float.MIN_VALUE (the smallest positive subnormal) and Float.MAX_VALUE. + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::denorm_min()), + "1.4E-45"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::max()), + "3.4028235E38"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(1.0F), "1.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(2.5F), "2.5"); +} + +TEST(VariantJsonUtilsTest, AppendEscapedJsonControlChars) { + // Control characters without a named escape are emitted as \uXXXX. + std::string out; + VariantJsonUtils::AppendEscapedJson(std::string_view("\x01\x1f", 2), &out); + EXPECT_EQ(out, "\"\\u0001\\u001f\""); +} + +TEST(VariantJsonUtilsTest, DateToStringNegativeYear) { + // A date far before the epoch renders a negative (BCE) year with a leading '-'. + std::string result = VariantJsonUtils::DateToString(-800000); + ASSERT_FALSE(result.empty()); + EXPECT_EQ(result.front(), '-'); +} + +TEST(VariantJsonUtilsTest, TimestampToStringEdgeCases) { + // Sub-second fraction trailing zeros are trimmed. + EXPECT_EQ(VariantJsonUtils::TimestampToString(500000, 0, /*with_offset=*/false), + "1970-01-01 00:00:00.5"); + // Negative epoch micros floor to the previous day (FloorDiv keeps a non-negative remainder). + EXPECT_EQ(VariantJsonUtils::TimestampToString(-1, 0, /*with_offset=*/false), + "1969-12-31 23:59:59.999999"); + // A positive zone offset is appended as +HH:MM. + EXPECT_EQ(VariantJsonUtils::TimestampToString(0, 8 * 3600, /*with_offset=*/true), + "1970-01-01 08:00:00+08:00"); +} + +TEST(VariantJsonUtilsTest, ZoneOffsetParsing) { + // `UTC`/`GMT`/`UT` prefixes are stripped before the fixed offset is parsed. + ASSERT_OK_AND_ASSIGN(int32_t utc_prefixed, + VariantJsonUtils::GetZoneOffsetSeconds("UTC+08:00", 0)); + EXPECT_EQ(utc_prefixed, 8 * 3600); + ASSERT_OK_AND_ASSIGN(int32_t ut_prefixed, VariantJsonUtils::GetZoneOffsetSeconds("UT+05", 0)); + EXPECT_EQ(ut_prefixed, 5 * 3600); + // HHMMSS form and a negative offset. + ASSERT_OK_AND_ASSIGN(int32_t hms, VariantJsonUtils::GetZoneOffsetSeconds("+18:30:15", 0)); + EXPECT_EQ(hms, 18 * 3600 + 30 * 60 + 15); + ASSERT_OK_AND_ASSIGN(int32_t neg, VariantJsonUtils::GetZoneOffsetSeconds("-06:30", 0)); + EXPECT_EQ(neg, -(6 * 3600 + 30 * 60)); + // Invalid forms are rejected: a non-digit char, a wrong digit count, and an out-of-range hour. + ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+9A", 0)); + ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+123", 0)); + ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+19:00", 0)); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_path_segment.cpp b/src/paimon/common/data/variant/variant_path_segment.cpp new file mode 100644 index 00000000..2dbf6d8d --- /dev/null +++ b/src/paimon/common/data/variant/variant_path_segment.cpp @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_path_segment.h" + +#include +#include + +#include "fmt/format.h" + +namespace paimon { + +namespace { + +// Matches `[123]` at the beginning of `remaining` and consumes it. +bool TryParseIndex(std::string_view* remaining, int32_t* index) { + std::string_view s = *remaining; + if (s.empty() || s[0] != '[') { + return false; + } + size_t i = 1; + int64_t value = 0; + size_t digits = 0; + while (i < s.size() && s[i] >= '0' && s[i] <= '9') { + value = value * 10 + (s[i] - '0'); + if (value > std::numeric_limits::max()) { + return false; + } + ++i; + ++digits; + } + if (digits == 0 || i >= s.size() || s[i] != ']') { + return false; + } + *index = static_cast(value); + remaining->remove_prefix(i + 1); + return true; +} + +// Matches `['key']` or `["key"]` at the beginning of `remaining` and consumes it. +bool TryParseQuotedKey(std::string_view* remaining, std::string* key) { + std::string_view s = *remaining; + if (s.size() < 4 || s[0] != '[' || (s[1] != '\'' && s[1] != '"')) { + return false; + } + char quote = s[1]; + size_t end = s.find(quote, 2); + if (end == std::string_view::npos || end == 2 || end + 1 >= s.size() || s[end + 1] != ']') { + return false; + } + key->assign(s.substr(2, end - 2)); + remaining->remove_prefix(end + 2); + return true; +} + +// Matches `.key` (one or more characters that are neither `.` nor `[`) at the beginning of +// `remaining` and consumes it. +bool TryParseDotKey(std::string_view* remaining, std::string* key) { + std::string_view s = *remaining; + if (s.size() < 2 || s[0] != '.') { + return false; + } + size_t end = 1; + while (end < s.size() && s[end] != '.' && s[end] != '[') { + ++end; + } + if (end == 1) { + return false; + } + key->assign(s.substr(1, end - 1)); + remaining->remove_prefix(end); + return true; +} + +} // namespace + +Result> VariantPathSegment::Parse(const std::string& path) { + // Mirrors the Java parser: the root `$` is located with a find, so segments are parsed + // after the FIRST `$` and any prefix before it is ignored. + size_t root = path.find('$'); + if (path.empty() || root == std::string::npos) { + return Status::Invalid(fmt::format("Invalid path: {}", path)); + } + std::string_view remaining = std::string_view(path).substr(root + 1); + std::vector segments; + while (!remaining.empty()) { + int32_t index = 0; + if (TryParseIndex(&remaining, &index)) { + segments.push_back(ArrayExtraction(index)); + continue; + } + std::string key; + if (TryParseDotKey(&remaining, &key) || TryParseQuotedKey(&remaining, &key)) { + segments.push_back(ObjectExtraction(std::move(key))); + continue; + } + return Status::Invalid(fmt::format("Invalid path: {}", path)); + } + return segments; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_path_segment.h b/src/paimon/common/data/variant/variant_path_segment.h new file mode 100644 index 00000000..fa012d7e --- /dev/null +++ b/src/paimon/common/data/variant/variant_path_segment.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// A path segment for variant get, representing either an object key access or an array index +/// access. +struct VariantPathSegment { + enum class Kind { kObjectExtraction, kArrayExtraction }; + + Kind kind; + /// The object key when `kind` is `kObjectExtraction`. + std::string key; + /// The array index when `kind` is `kArrayExtraction`. + int32_t index = 0; + + static VariantPathSegment ObjectExtraction(std::string key) { + VariantPathSegment segment; + segment.kind = Kind::kObjectExtraction; + segment.key = std::move(key); + return segment; + } + + static VariantPathSegment ArrayExtraction(int32_t index) { + VariantPathSegment segment; + segment.kind = Kind::kArrayExtraction; + segment.index = index; + return segment; + } + + /// Parses a path starting with `$`. Supported segments after the root are `.key`, `['key']`, + /// `["key"]` and `[index]`, e.g. `$.user.addresses[0]['city']`. + static Result> Parse(const std::string& path); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_reassembler.cpp b/src/paimon/common/data/variant/variant_reassembler.cpp new file mode 100644 index 00000000..7d301ff7 --- /dev/null +++ b/src/paimon/common/data/variant/variant_reassembler.cpp @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_reassembler.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/memory_pool.h" + +namespace paimon { + +namespace { + +// A row view over a shredded struct array, the Arrow analog of the Java `ShreddedRow`. +struct Cursor { + const arrow::StructArray* array; + int64_t row; + + bool IsNullAt(int32_t field) const { + return array->field(field)->IsNull(row); + } + + std::string_view GetBinary(int32_t field) const { + return static_cast(*array->field(field)).GetView(row); + } +}; + +Status Rebuild(const Cursor& cursor, std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder); + +Status RebuildTypedScalar(const Cursor& cursor, const VariantSchema& schema, + VariantBuilder* builder) { + int32_t typed_idx = schema.typed_idx; + const arrow::Array& typed_array = *cursor.array->field(typed_idx); + int64_t row = cursor.row; + const VariantSchema::ScalarType& scalar = schema.scalar_schema.value(); + switch (scalar.kind) { + case VariantSchema::ScalarKind::kString: + return builder->AppendString( + static_cast(typed_array).GetView(row)); + case VariantSchema::ScalarKind::kByte: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kShort: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kInt: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kLong: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kFloat: + return builder->AppendFloat( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kDouble: + return builder->AppendDouble( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kBoolean: + return builder->AppendBoolean( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kBinary: + return builder->AppendBinary( + static_cast(typed_array).GetView(row)); + case VariantSchema::ScalarKind::kDecimal: { + const auto& decimal_array = static_cast(typed_array); + arrow::Decimal128 value(decimal_array.GetValue(row)); + VariantDecimal decimal; + decimal.unscaled = (static_cast<__int128_t>(value.high_bits()) << 64) | + static_cast<__int128_t>(static_cast<__uint128_t>(value.low_bits())); + decimal.scale = scalar.scale; + return builder->AppendDecimal(decimal); + } + case VariantSchema::ScalarKind::kDate: + return builder->AppendDate( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kTimestampLtz: + return builder->AppendTimestamp( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kTimestampNtz: + return builder->AppendTimestampNtz( + static_cast(typed_array).Value(row)); + default: + return Status::NotImplemented("unsupported variant scalar kind in reassembly"); + } +} + +// Rebuilds a variant value from the shredded data according to the reconstruction algorithm in +// the parquet-format VariantShredding.md specification, appending the result to `builder`. +Status Rebuild(const Cursor& cursor, std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder) { + int32_t typed_idx = schema.typed_idx; + int32_t variant_idx = schema.variant_idx; + if (typed_idx >= 0 && !cursor.IsNullAt(typed_idx)) { + if (schema.scalar_schema.has_value()) { + return RebuildTypedScalar(cursor, schema, builder); + } else if (schema.array_schema != nullptr) { + const auto& list_array = + static_cast(*cursor.array->field(typed_idx)); + const auto& element_array = + static_cast(*list_array.values()); + int64_t element_start = list_array.value_offset(cursor.row); + int64_t element_end = list_array.value_offset(cursor.row + 1); + int32_t start = builder->GetWritePos(); + std::vector offsets; + offsets.reserve(element_end - element_start); + for (int64_t i = element_start; i < element_end; ++i) { + offsets.push_back(builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK(Rebuild(Cursor{&element_array, i}, metadata, + *schema.array_schema, pool, builder)); + } + return builder->FinishWritingArray(start, offsets); + } else { + const auto& object_array = + static_cast(*cursor.array->field(typed_idx)); + Cursor object_cursor{&object_array, cursor.row}; + std::vector fields; + int32_t start = builder->GetWritePos(); + for (size_t field_idx = 0; field_idx < schema.object_schema.size(); ++field_idx) { + // Shredded fields must not be null. + if (object_cursor.IsNullAt(static_cast(field_idx))) { + return VariantBinaryUtil::MalformedVariant( + "a shredded object field group is null"); + } + const std::string& field_name = schema.object_schema[field_idx].name; + const VariantSchema& field_schema = *schema.object_schema[field_idx].schema; + const auto& field_array = static_cast( + *object_array.field(static_cast(field_idx))); + Cursor field_cursor{&field_array, cursor.row}; + // If the field doesn't have a non-null `typed_value` or `value`, it is missing. + if ((field_schema.typed_idx >= 0 && + !field_cursor.IsNullAt(field_schema.typed_idx)) || + (field_schema.variant_idx >= 0 && + !field_cursor.IsNullAt(field_schema.variant_idx))) { + int32_t id = builder->AddKey(field_name); + fields.emplace_back(field_name, id, builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK( + Rebuild(field_cursor, metadata, field_schema, pool, builder)); + } + } + if (variant_idx >= 0 && !cursor.IsNullAt(variant_idx)) { + // Add the leftover fields in the variant binary. + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr leftover, + GenericVariant::Create(cursor.GetBinary(variant_idx), metadata, pool)); + PAIMON_ASSIGN_OR_RAISE(VariantValueType leftover_type, leftover->GetType()); + if (leftover_type != VariantValueType::kObject) { + return VariantBinaryUtil::MalformedVariant( + "the value column of a shredded object is not an object"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t leftover_size, leftover->ObjectSize()); + for (int32_t i = 0; i < leftover_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + leftover->GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant( + "a leftover object field is missing"); + } + // `value` must not contain any shredded field. + if (schema.object_schema_map.count(field->key) > 0) { + return VariantBinaryUtil::MalformedVariant(fmt::format( + "the value column duplicates the shredded field '{}'", field->key)); + } + int32_t id = builder->AddKey(field->key); + fields.emplace_back(field->key, id, builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK(builder->AppendVariant(*field->value)); + } + } + return builder->FinishWritingObject(start, &fields); + } + } else if (variant_idx >= 0 && !cursor.IsNullAt(variant_idx)) { + // `typed_value` doesn't exist or is null. Read from `value`. + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(cursor.GetBinary(variant_idx), metadata, pool)); + return builder->AppendVariant(*variant); + } else { + // The variant is missing in a context where it must be present; the data is invalid. + return VariantBinaryUtil::MalformedVariant( + "both typed_value and value of a required variant are null"); + } +} + +} // namespace + +Status VariantReassembler::RebuildValue(const arrow::StructArray& shredded, int64_t row, + std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, + VariantBuilder* builder) { + return Rebuild(Cursor{&shredded, row}, metadata, schema, pool, builder); +} + +Result> VariantReassembler::AssembleVariantArray( + const std::shared_ptr& shredded, + const std::shared_ptr& schema, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool) { + if (schema->top_level_metadata_idx < 0) { + return VariantBinaryUtil::MalformedVariant("a shredded file column misses metadata"); + } + auto output_type = VariantTypeUtils::UnshreddedStructType(); + std::unique_ptr output_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(arrow_pool, output_type, &output_builder)); + auto* struct_builder = static_cast(output_builder.get()); + auto* value_builder = static_cast(struct_builder->field_builder(0)); + auto* metadata_builder = static_cast(struct_builder->field_builder(1)); + + bool unshredded = schema->IsUnshredded(); + for (int64_t row = 0; row < shredded->length(); ++row) { + if (shredded->IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + Cursor cursor{shredded.get(), row}; + if (cursor.IsNullAt(schema->top_level_metadata_idx)) { + return VariantBinaryUtil::MalformedVariant("the variant metadata column is null"); + } + std::string_view metadata = cursor.GetBinary(schema->top_level_metadata_idx); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + if (unshredded) { + // Rebuilding is unnecessary for unshredded variants. + // TODO(nicholas): avoid copying the value/metadata binaries through the builder for + // unshredded files; the physical arrays could be returned directly with at most a + // per-row malformed-variant check. + if (cursor.IsNullAt(schema->variant_idx)) { + return VariantBinaryUtil::MalformedVariant( + "the value column of an unshredded variant is null"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + value_builder->Append(cursor.GetBinary(schema->variant_idx))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(metadata)); + } else { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(Rebuild(cursor, metadata, *schema, pool, &builder)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, builder.Build(pool)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(variant->RawValue())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(variant->Metadata())); + } + } + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(output_builder->Finish(&result)); + return result; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_reassembler.h b/src/paimon/common/data/variant/variant_reassembler.h new file mode 100644 index 00000000..ac5d9d5a --- /dev/null +++ b/src/paimon/common/data/variant/variant_reassembler.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class MemoryPool; +class StructArray; +} // namespace arrow + +namespace paimon { + +class VariantBuilder; + +/// Reassembles shredded variant columns back into the unshredded +/// `struct` representation, implementing the reconstruction +/// algorithm of the parquet-format VariantShredding.md specification (mirroring the Java +/// `ShreddingUtils.rebuild`). +class VariantReassembler { + public: + VariantReassembler() = delete; + ~VariantReassembler() = delete; + + /// Reassembles a shredded variant column into a `struct` array. + /// + /// @param shredded The physical shredded array read from the file. + /// @param schema The shredding schema of the column + /// (`VariantShreddingUtils::BuildVariantSchema` of the file type). + /// @param pool The memory pool used for intermediate variant rebuilding. + /// @param arrow_pool The Arrow memory pool used for the output array. + /// @return The unshredded variant array (`VariantTypeUtils::UnshreddedStructType`). + static Result> AssembleVariantArray( + const std::shared_ptr& shredded, + const std::shared_ptr& schema, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool); + + /// Rebuilds the variant value at `row` of a shredded (sub-)struct into `builder`, following + /// the same reconstruction algorithm. `schema` describes `shredded`, which may be any level + /// of the shredded tree; `metadata` is the column's top-level metadata binary. + static Status RebuildValue(const arrow::StructArray& shredded, int64_t row, + std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_schema.h b/src/paimon/common/data/variant/variant_schema.h new file mode 100644 index 00000000..d796c4a6 --- /dev/null +++ b/src/paimon/common/data/variant/variant_schema.h @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace paimon { + +/// Defines a valid shredding schema, as described in the parquet-format VariantShredding.md +/// specification. A shredding schema contains a `value` and an optional `typed_value` field. If a +/// `typed_value` is an array or struct, it recursively contains its own shredding schema for +/// elements and fields, respectively. The schema also contains a `metadata` field at the top +/// level, but not in recursively shredded fields. +class VariantSchema { + public: + enum class ScalarKind { + kBoolean, + kByte, + kShort, + kInt, + kLong, + kFloat, + kDouble, + kString, + kBinary, + kDecimal, + kDate, + kTimestampLtz, + kTimestampNtz, + kUuid, + }; + + struct ScalarType { + ScalarKind kind; + // Only meaningful when `kind` is `kDecimal`. + int32_t precision = 0; + int32_t scale = 0; + }; + + /// Represents one field of an object in the shredding schema. + struct ObjectField { + std::string name; + std::shared_ptr schema; + }; + + /// The index of the typed_value, value, and metadata fields in the schema, respectively. If a + /// given field is not in the schema, its value must be set to -1 to indicate that it is + /// invalid. The indices of valid fields are contiguous and start from 0. + int32_t typed_idx = -1; + int32_t variant_idx = -1; + /// Must be non-negative in the top-level schema, and -1 at all other nesting levels. + int32_t top_level_metadata_idx = -1; + /// The number of fields in the schema, i.e. a value between 1 and 3, depending on which of + /// value, typed_value and metadata are present. + int32_t num_fields = 0; + + /// Exactly one of the following describes typed_value (or none if there is no typed_value). + std::optional scalar_schema; + bool has_object_schema = false; + std::vector object_schema; + /// Fast lookup of object fields by name; values are indices into `object_schema`. + std::unordered_map object_schema_map; + std::shared_ptr array_schema; + + /// Whether the variant column is unshredded. The user is not required to do anything special, + /// but can have certain optimizations for unshredded variants. + bool IsUnshredded() const { + return top_level_metadata_idx >= 0 && variant_idx >= 0 && typed_idx < 0; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp b/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp new file mode 100644 index 00000000..a2cb234e --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/util/bitmap_ops.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_writer.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +/// Whether any enclosing struct is null at `row`. The contents of child slots under a null +/// ancestor are unspecified in Arrow and must not be decoded. +bool AnyAncestorNull(const std::vector& ancestors, int64_t row) { + for (const arrow::Array* ancestor : ancestors) { + if (ancestor->IsNull(row)) { + return true; + } + } + return false; +} + +/// Shreds one variant column array into its physical shredded representation. +Result> ShredVariantColumn( + const arrow::Array& column, const std::string& field_name, + const std::vector& ancestors, + const std::shared_ptr& variant_schema, + const std::shared_ptr& physical_type, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool) { + if (column.type_id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("variant column {} is not a struct column", field_name)); + } + const auto& variant_column = arrow::internal::checked_cast(column); + if (variant_column.num_fields() != 2) { + return Status::Invalid( + fmt::format("variant column {} is not a struct column", field_name)); + } + const auto& value_column = + arrow::internal::checked_cast(*variant_column.field(0)); + const auto& metadata_column = + arrow::internal::checked_cast(*variant_column.field(1)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + VariantShreddedColumnWriter::Create(variant_schema, physical_type, arrow_pool)); + for (int64_t row = 0; row < variant_column.length(); ++row) { + if (variant_column.IsNull(row) || AnyAncestorNull(ancestors, row)) { + PAIMON_RETURN_NOT_OK(writer->AppendNull()); + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(value_column.GetView(row), metadata_column.GetView(row), pool)); + PAIMON_RETURN_NOT_OK(writer->Append(*variant)); + } + return writer->Finish(); +} + +} // namespace + +VariantShreddingBatchConverter::VariantShreddingBatchConverter( + const std::shared_ptr& plan, const std::shared_ptr& pool) + : plan_(plan), pool_(pool), arrow_pool_(GetArrowPool(pool)) {} + +Result> VariantShreddingBatchConverter::Create( + const std::shared_ptr& plan, + const std::shared_ptr& pool) { + if (!plan) { + return Status::Invalid("variant shredding batch converter requires a write plan"); + } + return std::shared_ptr( + new VariantShreddingBatchConverter(plan, pool)); +} + +const std::shared_ptr& VariantShreddingBatchConverter::GetPhysicalSchema() const { + return plan_->PhysicalSchema(); +} + +Result> VariantShreddingBatchConverter::ConvertField( + const std::shared_ptr& logical, + const std::shared_ptr& logical_field, + const std::shared_ptr& physical_field, std::vector* path, + std::vector* ancestors) const { + for (const auto& column : plan_->Columns()) { + if (column.path == *path) { + return ShredVariantColumn(*logical, logical_field->name(), *ancestors, + column.variant_schema, column.physical_type, pool_, + arrow_pool_.get()); + } + } + if (logical_field->type()->Equals(*physical_field->type())) { + return logical; + } + // The types differ below this struct field: convert the planned descendants recursively and + // rebuild the struct with its original validity. + if (logical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("variant shredding cannot convert non-struct field {}", + logical_field->name())); + } + const auto& logical_struct = arrow::internal::checked_cast(*logical); + const auto& logical_type = + arrow::internal::checked_cast(*logical_field->type()); + const auto& physical_type = + arrow::internal::checked_cast(*physical_field->type()); + if (logical_type.num_fields() != physical_type.num_fields()) { + return Status::Invalid(fmt::format("variant shredding physical struct {} does not match", + physical_field->name())); + } + arrow::ArrayVector converted_children(logical_struct.num_fields()); + ancestors->push_back(logical.get()); + for (int32_t i = 0; i < logical_struct.num_fields(); ++i) { + path->push_back(i); + PAIMON_ASSIGN_OR_RAISE(converted_children[i], + ConvertField(logical_struct.field(i), logical_type.field(i), + physical_type.field(i), path, ancestors)); + path->pop_back(); + } + ancestors->pop_back(); + std::shared_ptr validity; + int64_t null_count = logical_struct.null_count(); + if (null_count > 0) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + validity, + arrow::internal::CopyBitmap(arrow_pool_.get(), logical_struct.null_bitmap_data(), + logical_struct.offset(), logical_struct.length())); + } + return std::make_shared(physical_field->type(), logical_struct.length(), + std::move(converted_children), std::move(validity), + null_count); +} + +Result> VariantShreddingBatchConverter::Convert( + ArrowArray* logical_batch) { + auto logical_struct_type = arrow::struct_(plan_->LogicalSchema()->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_array, + arrow::ImportArray(logical_batch, logical_struct_type)); + const auto& logical_struct = std::static_pointer_cast(logical_array); + + const auto& logical_fields = plan_->LogicalSchema()->fields(); + const auto& physical_fields = plan_->PhysicalSchema()->fields(); + arrow::ArrayVector physical_arrays(logical_struct->num_fields()); + std::vector path; + std::vector ancestors; + for (int32_t i = 0; i < logical_struct->num_fields(); ++i) { + path.push_back(i); + PAIMON_ASSIGN_OR_RAISE(physical_arrays[i], + ConvertField(logical_struct->field(i), logical_fields[i], + physical_fields[i], &path, &ancestors)); + path.pop_back(); + } + + arrow::FieldVector physical_struct_fields(physical_fields.begin(), physical_fields.end()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr physical_struct, + arrow::StructArray::Make(physical_arrays, physical_struct_fields)); + auto result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*physical_struct, result.get())); + return result; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_batch_converter.h b/src/paimon/common/data/variant/variant_shredding_batch_converter.h new file mode 100644 index 00000000..706af162 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_batch_converter.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/common/data/shredding/shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +struct ArrowArray; + +namespace arrow { +class Array; +class Field; +class MemoryPool; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Converts logical batches containing VARIANT columns into physical batches where each planned +/// variant column is replaced by its shredded struct representation. Unplanned columns are +/// passed through unchanged. +class VariantShreddingBatchConverter : public ShreddingBatchConverter { + public: + static Result> Create( + const std::shared_ptr& plan, + const std::shared_ptr& pool); + + /// The physical schema produced by this converter. + const std::shared_ptr& GetPhysicalSchema() const override; + + /// Converts a logical batch to a physical batch. + /// @param logical_batch Input ArrowArray (C ABI) with the logical schema. Consumed on + /// success. + /// @return Owned physical ArrowArray (C ABI) with the physical schema. + Result> Convert(ArrowArray* logical_batch) override; + + private: + VariantShreddingBatchConverter(const std::shared_ptr& plan, + const std::shared_ptr& pool); + + /// Converts the logical array at field-index path `path`, shredding it when planned and + /// otherwise recursing into struct children whose subtree contains a planned column. + /// `ancestors` holds the enclosing struct arrays; rows that are null at any level shred to + /// null without decoding the (unspecified) child slot contents. + Result> ConvertField( + const std::shared_ptr& logical, + const std::shared_ptr& logical_field, + const std::shared_ptr& physical_field, std::vector* path, + std::vector* ancestors) const; + + std::shared_ptr plan_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_inference_session.cpp b/src/paimon/common/data/variant/variant_shredding_inference_session.cpp new file mode 100644 index 00000000..5b24e046 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_inference_session.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_inference_session.h" + +#include + +namespace paimon { + +VariantShreddingInferenceSession::VariantShreddingInferenceSession( + InferVariantShreddingSchema inferrer, int32_t effective_sample_size, double admission_ratio, + double retention_ratio) + : inferrer_(std::move(inferrer)), + effective_sample_size_(effective_sample_size), + admission_ratio_(admission_ratio), + retention_ratio_(retention_ratio) {} + +Result> VariantShreddingInferenceSession::InferSchema( + const InferVariantShreddingSchema::SampleBatches& samples) { + InferVariantShreddingSchema::AdaptiveInferenceResult result; + if (!has_committed_evidence_) { + PAIMON_ASSIGN_OR_RAISE(result, inferrer_.InferInitial(samples, effective_sample_size_)); + } else { + PAIMON_ASSIGN_OR_RAISE( + result, + inferrer_.InferAdaptive(committed_evidence_, committed_selected_schemas_, samples, + effective_sample_size_, admission_ratio_, retention_ratio_)); + } + std::shared_ptr physical_schema = result.physical_schema; + pending_result_ = std::move(result); + return physical_schema; +} + +Status VariantShreddingInferenceSession::CommitPendingInference() { + if (!pending_result_.has_value()) { + return Status::Invalid("No pending Variant inference to commit."); + } + committed_evidence_ = std::move(pending_result_->evidence); + committed_selected_schemas_ = std::move(pending_result_->selected_schemas); + pending_result_.reset(); + has_committed_evidence_ = true; + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_inference_session.h b/src/paimon/common/data/variant/variant_shredding_inference_session.h new file mode 100644 index 00000000..d0de3b93 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_inference_session.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" +#include "paimon/result.h" + +namespace paimon { + +/// Rolling-writer-scoped Variant inference state. Evidence is bounded and is committed only +/// after the corresponding file has closed successfully. +class VariantShreddingInferenceSession { + public: + VariantShreddingInferenceSession(InferVariantShreddingSchema inferrer, + int32_t effective_sample_size, double admission_ratio, + double retention_ratio); + + bool HasPrior() const { + return has_committed_evidence_; + } + + /// Infers one complete physical row schema and retains the corresponding evidence as pending + /// state until the file using this schema has completed successfully. + Result> InferSchema( + const InferVariantShreddingSchema::SampleBatches& samples); + + Status CommitPendingInference(); + + private: + InferVariantShreddingSchema inferrer_; + int32_t effective_sample_size_; + double admission_ratio_; + double retention_ratio_; + + bool has_committed_evidence_ = false; + InferVariantShreddingSchema::InferenceEvidence committed_evidence_; + InferVariantShreddingSchema::SelectedSchemas committed_selected_schemas_; + std::optional pending_result_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp new file mode 100644 index 00000000..b1e14b1a --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_get.h" +#include "paimon/common/data/variant/variant_reassembler.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +/// Reassembles the full variant of a shredded file column back into +/// `struct` (a plain VARIANT read). +class FullVariantColumnReadPlan : public ShreddingColumnReadPlan { + public: + FullVariantColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + std::shared_ptr schema, + std::shared_ptr pool) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + schema_(std::move(schema)), + pool_(std::move(pool)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + if (physical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("cannot cast shredded variant field {} to a struct", + physical_field_->name())); + } + auto physical_struct = std::static_pointer_cast(physical); + return VariantReassembler::AssembleVariantArray(physical_struct, schema_, pool_, pool); + } + + private: + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + std::shared_ptr schema_; + std::shared_ptr pool_; +}; + +/// Restores the logical `struct` field order of an untyped physical +/// `struct` without copying either binary child. +class UntypedVariantColumnReadPlan : public ShreddingColumnReadPlan { + public: + UntypedVariantColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field) + : logical_field_(std::move(logical_field)), physical_field_(std::move(physical_field)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool*) const override { + if (physical->type_id() != arrow::Type::STRUCT || + physical->data()->child_data.size() != 2) { + return Status::Invalid(fmt::format("cannot reorder untyped physical variant field {}", + physical_field_->name())); + } + std::shared_ptr logical_data = physical->data()->Copy(); + logical_data->type = logical_field_->type(); + std::swap(logical_data->child_data[0], logical_data->child_data[1]); + return arrow::MakeArray(std::move(logical_data)); + } + + private: + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; +}; + +/// A node of a nested variant plan tree: a variant position with its own leaf plan, or a nested +/// container level to descend through. +struct NestedVariantNode { + /// The leaf plan when this position is a variant column: a full reassembly or a + /// variant-access extraction. + std::shared_ptr plan; + /// The children whose subtree holds planned variants, by Arrow child index. The index + /// addresses struct fields, the list element field, and the map entries field alike. + std::map children; +}; + +/// Applies the leaf plans of the variant columns nested inside a top-level STRUCT / LIST / MAP +/// column, rebuilding the containers around them. +class NestedVariantColumnReadPlan : public ShreddingColumnReadPlan { + public: + NestedVariantColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + NestedVariantNode root) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + root_(std::move(root)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + return AssembleNode(physical, logical_field_, root_, pool); + } + + private: + Result> AssembleNode( + const std::shared_ptr& physical, + const std::shared_ptr& logical_field, const NestedVariantNode& node, + arrow::MemoryPool* pool) const { + if (node.plan != nullptr) { + return node.plan->Assemble(physical, pool); + } + const std::shared_ptr& logical_type = logical_field->type(); + const std::shared_ptr& physical_data = physical->data(); + if (physical->type_id() != logical_type->id()) { + return Status::Invalid(fmt::format( + "shredded variant field {} is stored as {} but read as {}", logical_field->name(), + physical->type()->ToString(), logical_type->ToString())); + } + // Swap the planned children into a copy of the physical array's own data: the validity + // bitmap, list offsets and slice offset carry over untouched, so STRUCT, LIST and MAP + // rebuild alike. Children are assembled unsliced to stay aligned with those offsets. + std::shared_ptr data = physical_data->Copy(); + data->type = logical_type; + for (const auto& [index, child_node] : node.children) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + AssembleNode(arrow::MakeArray(physical_data->child_data[index]), + logical_type->field(index), child_node, pool)); + data->child_data[index] = child->data(); + } + return arrow::MakeArray(data); + } + + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + NestedVariantNode root_; +}; + +/// Whether the field is read as a variant column: either as a plain VARIANT or as a +/// variant-access projection (which keeps the variant marker but replaces the type). +bool IsVariantReadField(const std::shared_ptr& field) { + return VariantTypeUtils::IsVariantField(field) || + VariantAccessUtils::IsVariantAccessType(field->type()); +} + +/// Whether the type is a nested container the plan tree descends through. +bool IsNestedContainer(const std::shared_ptr& type) { + return type->id() == arrow::Type::STRUCT || type->id() == arrow::Type::LIST || + type->id() == arrow::Type::MAP; +} + +/// Whether the field is a nested container (that is not itself a variant) holding a variant +/// field in its subtree. +bool ContainsNestedVariant(const std::shared_ptr& field) { + if (IsVariantReadField(field) || !IsNestedContainer(field->type())) { + return false; + } + for (const auto& child : field->type()->fields()) { + if (IsVariantReadField(child) || ContainsNestedVariant(child)) { + return true; + } + } + return false; +} + +// Both are defined below: CreateVariantColumnPlan after the access plan it builds, and +// BuildNestedVariantPlan because it and PlanChild are mutually recursive. +Result> CreateVariantColumnPlan( + const std::shared_ptr& read_field, + const std::shared_ptr& file_field, const std::shared_ptr& pool, + bool allow_pruning); + +Result BuildNestedVariantPlan(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_field, + NestedVariantNode* node); + +/// Plans one child position: a variant leaf, or a nested container to descend into. Returns +/// whether the position needs a plan; `physical_child` receives the field to push down for it. +Result PlanChild(const std::shared_ptr& read_child, + const std::shared_ptr& file_child, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_child, NestedVariantNode* node) { + *physical_child = read_child; + if (IsVariantReadField(read_child)) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + CreateVariantColumnPlan(read_child, file_child, pool, !inside_repeated)); + if (plan == nullptr) { + // An unshredded column read as a plain VARIANT needs no plan. + return false; + } + *physical_child = read_child->WithType(plan->PhysicalField()->type()); + node->plan = std::move(plan); + return true; + } + if (!ContainsNestedVariant(read_child) || + file_child->type()->id() != read_child->type()->id()) { + return false; + } + return BuildNestedVariantPlan(read_child, file_child, pool, inside_repeated, physical_child, + node); +} + +/// Whether the read and file children of a level inside a repeated group line up one to one, +/// which they must because the file subtree is pushed down verbatim there. +/// +/// Only STRUCT children are matched by name: LIST and MAP child names are format conventions +/// that differ between the read schema and the file (`element` vs `item`, `entries` vs the +/// parquet `key_value` group), so those match positionally. +bool ChildrenLineUp(const arrow::DataType& read_type, const arrow::DataType& file_type) { + if (read_type.num_fields() != file_type.num_fields()) { + return false; + } + if (read_type.id() != arrow::Type::STRUCT) { + return true; + } + for (int32_t i = 0; i < read_type.num_fields(); ++i) { + if (read_type.field(i)->name() != file_type.field(i)->name()) { + return false; + } + } + return true; +} + +/// Builds the nested plan of one column subtree: substitutes the physical file types at the +/// nested variant positions into `read_field` (producing the physical field to push down) and +/// records their leaf plans in `node`. Returns whether any nested position needs a plan. +/// +/// A STRUCT level is matched by field name and keeps per-child pruning. A LIST or MAP level is +/// matched positionally and pushes the file subtree down verbatim, because the parquet reader +/// rejects partial projection inside a repeated group; everything below it is therefore read in +/// full and only reassembled back. +Result BuildNestedVariantPlan(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_field, + NestedVariantNode* node) { + const arrow::DataType& read_type = *read_field->type(); + const arrow::DataType& file_type = *file_field->type(); + // True for a LIST or MAP level and for everything below it, including plain STRUCT levels. + const bool in_repeated_subtree = inside_repeated || read_type.id() != arrow::Type::STRUCT; + *physical_field = read_field; + if (in_repeated_subtree && !ChildrenLineUp(read_type, file_type)) { + return false; + } + + arrow::FieldVector physical_children = read_type.fields(); + bool needs_plan = false; + for (int32_t i = 0; i < read_type.num_fields(); ++i) { + const std::shared_ptr& read_child = read_type.field(i); + std::shared_ptr file_child = + in_repeated_subtree + ? file_type.field(i) + : arrow::internal::checked_cast(file_type).GetFieldByName( + read_child->name()); + if (file_child == nullptr) { + // The nested column is absent in the file (schema evolution); it is filled with + // nulls downstream. + continue; + } + NestedVariantNode child_node; + PAIMON_ASSIGN_OR_RAISE(bool child_needs_plan, + PlanChild(read_child, file_child, pool, in_repeated_subtree, + &physical_children[i], &child_node)); + if (child_needs_plan) { + node->children[i] = std::move(child_node); + needs_plan = true; + } + } + if (!needs_plan) { + return false; + } + *physical_field = + in_repeated_subtree ? file_field : read_field->WithType(arrow::struct_(physical_children)); + return true; +} + +/// One access path segment resolved against the shredded schema of one file. +struct ResolvedSegment { + VariantPathSegment raw; + bool is_object = false; + // The `typed_value` index at this level, or -1 when the path leaves the shredded schema + // here and continues inside the `value` binary. + int32_t typed_idx = -1; + // The object field index inside the typed object, or the array element index. + int32_t extraction_idx = -1; +}; + +struct ResolvedSpec { + VariantAccessSpec spec; + std::vector segments; +}; + +ResolvedSpec ResolveSpec(const VariantAccessSpec& spec, const VariantSchema* root) { + ResolvedSpec resolved; + resolved.spec = spec; + const VariantSchema* schema = root; + for (const auto& segment : spec.segments) { + ResolvedSegment r; + r.raw = segment; + if (segment.kind == VariantPathSegment::Kind::kObjectExtraction) { + r.is_object = true; + if (schema != nullptr && !schema->object_schema.empty()) { + auto it = schema->object_schema_map.find(segment.key); + if (it != schema->object_schema_map.end()) { + r.typed_idx = schema->typed_idx; + r.extraction_idx = it->second; + schema = schema->object_schema[it->second].schema.get(); + } else { + schema = nullptr; + } + } else { + schema = nullptr; + } + } else { + if (schema != nullptr && schema->array_schema != nullptr) { + r.typed_idx = schema->typed_idx; + r.extraction_idx = segment.index; + schema = schema->array_schema.get(); + } else { + schema = nullptr; + } + } + resolved.segments.push_back(std::move(r)); + } + return resolved; +} + +/// Extracts the paths described by a variant-access projection, reading typed sub-columns +/// directly and falling back to the `value` binary where the path is not shredded. +class VariantAccessColumnReadPlan : public ShreddingColumnReadPlan { + public: + VariantAccessColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + std::shared_ptr schema, + std::vector specs, std::shared_ptr pool) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + schema_(std::move(schema)), + specs_(std::move(specs)), + pool_(std::move(pool)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + if (physical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("cannot cast shredded variant field {} to a struct", + physical_field_->name())); + } + const auto& physical_struct = static_cast(*physical); + std::unique_ptr builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(pool, logical_field_->type(), &builder)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Reserve(physical_struct.length())); + auto* struct_builder = static_cast(builder.get()); + for (int64_t row = 0; row < physical_struct.length(); ++row) { + if (physical_struct.IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + if (physical_struct.field(schema_->top_level_metadata_idx)->IsNull(row)) { + return VariantBinaryUtil::MalformedVariant("the variant metadata column is null"); + } + std::string_view metadata = static_cast( + *physical_struct.field(schema_->top_level_metadata_idx)) + .GetView(row); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + for (size_t i = 0; i < specs_.size(); ++i) { + PAIMON_RETURN_NOT_OK( + ExtractField(physical_struct, row, metadata, specs_[i], + struct_builder->field_builder(static_cast(i)))); + } + } + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&result)); + return result; + } + + private: + Status ExtractField(const arrow::StructArray& root, int64_t root_row, std::string_view metadata, + const ResolvedSpec& resolved, arrow::ArrayBuilder* builder) const { + const arrow::StructArray* current = &root; + int64_t row = root_row; + const VariantSchema* schema = schema_.get(); + size_t segment_idx = 0; + while (segment_idx < resolved.segments.size()) { + const ResolvedSegment& segment = resolved.segments[segment_idx]; + if (segment.typed_idx < 0) { + // The path leaves the shredded schema here; walk the remaining raw path inside + // the `value` binary. + return ExtractFromBinary(*current, row, metadata, resolved, segment_idx, schema, + builder); + } + if (current->field(segment.typed_idx)->IsNull(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + if (segment.is_object) { + const auto& object_array = + static_cast(*current->field(segment.typed_idx)); + const auto& field_array = static_cast( + *object_array.field(segment.extraction_idx)); + if (field_array.IsNull(row)) { + // Shredded object fields must not be null. + return VariantBinaryUtil::MalformedVariant( + "a shredded object field group is null"); + } + schema = schema->object_schema[segment.extraction_idx].schema.get(); + current = &field_array; + // A field is missing when neither its typed_value nor its value is present. + bool typed_present = + schema->typed_idx >= 0 && !current->field(schema->typed_idx)->IsNull(row); + bool variant_present = + schema->variant_idx >= 0 && !current->field(schema->variant_idx)->IsNull(row); + if (!typed_present && !variant_present) { + return ToPaimonStatus(builder->AppendNull()); + } + } else { + const auto& list_array = + static_cast(*current->field(segment.typed_idx)); + if (segment.extraction_idx >= list_array.value_length(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + int64_t element_row = list_array.value_offset(row) + segment.extraction_idx; + const auto& element_array = + static_cast(*list_array.values()); + if (element_array.IsNull(element_row)) { + // Shredded array elements must not be null. + return VariantBinaryUtil::MalformedVariant( + "a shredded array element group is null"); + } + schema = schema->array_schema.get(); + current = &element_array; + row = element_row; + } + ++segment_idx; + } + + // The terminal position: rebuild the (sub-)variant and cast it to the target type. + if (schema->typed_idx >= 0 && !current->field(schema->typed_idx)->IsNull(row)) { + VariantBuilder variant_builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(VariantReassembler::RebuildValue(*current, row, metadata, *schema, + pool_, &variant_builder)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + variant_builder.Build(pool_)); + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + if (schema->variant_idx >= 0 && !current->field(schema->variant_idx)->IsNull(row)) { + std::string_view value = + static_cast(*current->field(schema->variant_idx)) + .GetView(row); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + return VariantBinaryUtil::MalformedVariant( + "both typed_value and value of a required variant are null"); + } + + Status ExtractFromBinary(const arrow::StructArray& current, int64_t row, + std::string_view metadata, const ResolvedSpec& resolved, + size_t segment_idx, const VariantSchema* schema, + arrow::ArrayBuilder* builder) const { + if (schema->variant_idx < 0 || current.field(schema->variant_idx)->IsNull(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + std::string_view value = + static_cast(*current.field(schema->variant_idx)) + .GetView(row); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + for (; segment_idx < resolved.segments.size() && variant != nullptr; ++segment_idx) { + const VariantPathSegment& raw = resolved.segments[segment_idx].raw; + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, variant->GetType()); + if (raw.kind == VariantPathSegment::Kind::kObjectExtraction && + type == VariantValueType::kObject) { + PAIMON_ASSIGN_OR_RAISE(variant, variant->GetFieldByKey(raw.key)); + } else if (raw.kind == VariantPathSegment::Kind::kArrayExtraction && + type == VariantValueType::kArray) { + PAIMON_ASSIGN_OR_RAISE(variant, variant->GetElementAtIndex(raw.index)); + } else { + variant = nullptr; + } + } + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + std::shared_ptr schema_; + std::vector specs_; + std::shared_ptr pool_; +}; + +/// Builds the leaf read plan of one variant position, at the top level or nested inside a +/// container column: a variant-access projection extracts the described paths (from a shredded +/// or an unshredded file column), and a plain VARIANT read reassembles a shredded file column. +/// Returns nullptr when a plain VARIANT read of an unshredded file column needs no plan. +/// +/// `allow_pruning` narrows the scan to the sub-columns the access paths need. It is off inside a +/// repeated group, where the file subtree must be read whole. +Result> CreateVariantColumnPlan( + const std::shared_ptr& read_field, + const std::shared_ptr& file_field, const std::shared_ptr& pool, + bool allow_pruning) { + if (VariantAccessUtils::IsVariantAccessType(read_field->type())) { + PAIMON_ASSIGN_OR_RAISE(std::vector specs, + VariantAccessUtils::ParseAccessSpecs(read_field)); + std::shared_ptr physical_field = file_field; + if (allow_pruning) { + PAIMON_ASSIGN_OR_RAISE(physical_field, + VariantAccessUtils::ClipShreddedFileField(specs, file_field)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical_field->type())); + std::vector resolved; + resolved.reserve(specs.size()); + for (const auto& spec : specs) { + resolved.push_back(ResolveSpec(spec, schema.get())); + } + return std::make_shared( + read_field, physical_field, std::move(schema), std::move(resolved), pool); + } + if (VariantShreddingUtils::IsUntypedPhysicalVariantType(file_field->type())) { + return std::make_shared(read_field, file_field); + } + if (!VariantShreddingUtils::IsShreddedFileType(file_field->type())) { + return std::shared_ptr(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(file_field->type())); + return std::make_shared(read_field, file_field, std::move(schema), + pool); +} + +} // namespace + +Result>> +VariantShreddingReadPlanFactory::CreateReadPlans(const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema, + const std::shared_ptr& pool) { + std::map> plans; + for (const auto& read_field : read_schema->fields()) { + bool nested_variant = ContainsNestedVariant(read_field); + if (!IsVariantReadField(read_field) && !nested_variant) { + continue; + } + auto file_field = file_schema->GetFieldByName(read_field->name()); + if (file_field == nullptr) { + // The column is absent in the file (schema evolution); it is filled with nulls + // downstream. + continue; + } + if (nested_variant) { + if (file_field->type()->id() != read_field->type()->id()) { + continue; + } + std::shared_ptr physical_field; + NestedVariantNode root; + PAIMON_ASSIGN_OR_RAISE( + bool needs_plan, + BuildNestedVariantPlan(read_field, file_field, pool, + /*inside_repeated=*/false, &physical_field, &root)); + if (needs_plan) { + plans.emplace(read_field->name(), std::make_shared( + read_field, physical_field, std::move(root))); + } + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + CreateVariantColumnPlan(read_field, file_field, pool, /*allow_pruning=*/true)); + if (plan != nullptr) { + plans.emplace(read_field->name(), std::move(plan)); + } + } + return plans; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h new file mode 100644 index 00000000..2df392d4 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_read_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +/// Builds per-column read plans for VARIANT columns: +/// - a plain VARIANT read of a shredded file reassembles the full variant; +/// - a variant-access projection (struct whose children carry `__VARIANT_METADATA` +/// descriptions) extracts the described paths, reading only the required shredded +/// sub-columns from a shredded file (or the binary from an unshredded file). +class VariantShreddingReadPlanFactory { + public: + VariantShreddingReadPlanFactory() = delete; + ~VariantShreddingReadPlanFactory() = delete; + + /// Creates the per-column read plans for the variant columns of `read_schema` against + /// `file_schema`; the map is empty when no plan applies (no shredded file column and no + /// variant-access projection). + static Result>> CreateReadPlans( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp new file mode 100644 index 00000000..61107e7d --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_shredding_writer.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantShreddingReadPlanFactoryTest : public ::testing::Test { + public: + std::shared_ptr Variant(const std::string& json) { + auto result = GenericVariant::FromJson(json, pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + // The full shredded StructArray (struct{metadata, value, typed_value}) for one variant, using + // the production writer. + void MakeFullShredded(const std::shared_ptr& logical, + const std::shared_ptr& variant, + std::shared_ptr* physical_type, + std::shared_ptr* array) { + ASSERT_OK_AND_ASSIGN(*physical_type, + VariantShreddingUtils::VariantShreddingSchema(logical)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(*physical_type)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + VariantShreddedColumnWriter::Create(schema, *physical_type, + arrow::default_memory_pool())); + ASSERT_OK(writer->Append(*variant)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr arr, writer->Finish()); + *array = std::static_pointer_cast(arr); + } + + // The unshredded physical StructArray (struct{value, metadata}) for one variant. + std::shared_ptr MakeUnshredded(const std::shared_ptr& v) { + auto value = v->Value(); + EXPECT_TRUE(value.ok()) << value.status().ToString(); + return MakeUnshreddedRaw(value.value(), v->Metadata(), /*metadata_null=*/false); + } + + std::shared_ptr MakeUnshreddedRaw(std::string_view value, + std::string_view metadata, + bool metadata_null) { + arrow::BinaryBuilder value_builder; + EXPECT_TRUE(value_builder.Append(value).ok()); + std::shared_ptr value_array; + EXPECT_TRUE(value_builder.Finish(&value_array).ok()); + arrow::BinaryBuilder metadata_builder; + if (metadata_null) { + EXPECT_TRUE(metadata_builder.AppendNull().ok()); + } else { + EXPECT_TRUE(metadata_builder.Append(metadata).ok()); + } + std::shared_ptr metadata_array; + EXPECT_TRUE(metadata_builder.Finish(&metadata_array).ok()); + auto made = arrow::StructArray::Make({value_array, metadata_array}, + std::vector{"value", "metadata"}); + EXPECT_TRUE(made.ok()) << made.status().ToString(); + return made.ValueOrDie(); + } + + // A variant-access child: an arrow field carrying a `__VARIANT_METADATA` description. + static std::shared_ptr AccessChild(const std::string& name, + const std::shared_ptr& type, + const std::string& path) { + return arrow::field(name, type, /*nullable=*/true, + arrow::key_value_metadata({DataField::DESCRIPTION}, + {VariantAccessUtils::BuildVariantMetadata( + path, /*fail_on_error=*/false, "UTC")})); + } + + std::shared_ptr CreatePlan( + const std::shared_ptr& read_field, + const std::shared_ptr& file_field) { + auto plans = VariantShreddingReadPlanFactory::CreateReadPlans( + arrow::schema({read_field}), arrow::schema({file_field}), pool_); + EXPECT_TRUE(plans.ok()) << plans.status().ToString(); + auto it = plans.value().find(read_field->name()); + EXPECT_NE(it, plans.value().end()); + return it->second; + } + + static std::shared_ptr Int32Array(int32_t value) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Append(value).ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(VariantShreddingReadPlanFactoryTest, FullVariantReadOfShreddedFile) { + std::shared_ptr variant = Variant(R"({"a": 5})"); + std::shared_ptr physical; + std::shared_ptr shredded; + MakeFullShredded(arrow::struct_({arrow::field("a", arrow::int64())}), variant, &physical, + &shredded); + ASSERT_FALSE(HasFatalFailure()); + + auto read_field = VariantTypeUtils::ToArrowField("v"); + auto file_field = arrow::field("v", physical); + std::shared_ptr plan = CreatePlan(read_field, file_field); + ASSERT_NE(plan, nullptr); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled, + plan->Assemble(shredded, arrow::default_memory_pool())); + auto assembled_struct = std::static_pointer_cast(assembled); + auto value_column = std::static_pointer_cast(assembled_struct->field(0)); + auto metadata_column = std::static_pointer_cast(assembled_struct->field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr rebuilt, + GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, rebuilt->ToJson()); + EXPECT_EQ(json, R"({"a":5})"); + + // Assembling a non-struct physical array is an error. + auto ints = Int32Array(1); + ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool())); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, FullVariantReadOfUntypedPhysicalFile) { + std::shared_ptr variant = Variant(R"({"a": 5})"); + std::shared_ptr physical; + std::shared_ptr shredded; + MakeFullShredded(arrow::null(), variant, &physical, &shredded); + ASSERT_FALSE(HasFatalFailure()); + + const auto& physical_struct = static_cast(*physical); + ASSERT_EQ(physical_struct.num_fields(), 2); + ASSERT_EQ(physical_struct.field(0)->name(), VariantDefs::kMetadataFieldName); + ASSERT_EQ(physical_struct.field(1)->name(), VariantDefs::kValueFieldName); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(physical)); + ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(physical)); + + auto read_field = VariantTypeUtils::ToArrowField("v"); + auto file_field = arrow::field("v", physical); + std::shared_ptr plan = CreatePlan(read_field, file_field); + ASSERT_NE(plan, nullptr); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled, + plan->Assemble(shredded, arrow::default_memory_pool())); + auto assembled_struct = std::static_pointer_cast(assembled); + ASSERT_EQ(assembled_struct->data()->child_data[0], shredded->data()->child_data[1]); + ASSERT_EQ(assembled_struct->data()->child_data[1], shredded->data()->child_data[0]); + auto value_column = std::static_pointer_cast(assembled_struct->field(0)); + auto metadata_column = std::static_pointer_cast(assembled_struct->field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr rebuilt, + GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, rebuilt->ToJson()); + EXPECT_EQ(json, R"({"a":5})"); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionOnUnshreddedFile) { + std::shared_ptr variant = Variant(R"({"a": 5, "b": [10, 20], "c": "hi"})"); + std::shared_ptr unshredded = MakeUnshredded(variant); + ASSERT_FALSE(HasFatalFailure()); + + auto read_field = + arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a"), + AccessChild("c", arrow::utf8(), "$.c"), + AccessChild("b0", arrow::int64(), "$.b[0]"), + AccessChild("missing", arrow::utf8(), "$.missing")})); + // Unshredded file column: struct{value, metadata}. + auto file_field = arrow::field("v", VariantTypeUtils::UnshreddedStructType()); + std::shared_ptr plan = CreatePlan(read_field, file_field); + ASSERT_NE(plan, nullptr); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled, + plan->Assemble(unshredded, arrow::default_memory_pool())); + auto row = std::static_pointer_cast(assembled); + ASSERT_EQ(row->length(), 1); + EXPECT_EQ(static_cast(*row->field(0)).Value(0), 5); + EXPECT_EQ(static_cast(*row->field(1)).GetString(0), "hi"); + EXPECT_EQ(static_cast(*row->field(2)).Value(0), 10); + EXPECT_TRUE(row->field(3)->IsNull(0)); + + // A non-struct physical array is an error. + auto ints = Int32Array(1); + ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool())); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionRejectsNullMetadata) { + std::shared_ptr variant = Variant(R"({"a": 5})"); + ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value()); + std::shared_ptr bad = + MakeUnshreddedRaw(value, variant->Metadata(), /*metadata_null=*/true); + ASSERT_FALSE(HasFatalFailure()); + + auto read_field = arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a")})); + auto file_field = arrow::field("v", VariantTypeUtils::UnshreddedStructType()); + std::shared_ptr plan = CreatePlan(read_field, file_field); + ASSERT_NE(plan, nullptr); + ASSERT_NOK(plan->Assemble(bad, arrow::default_memory_pool())); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionOnShreddedFile) { + // Extracting shredded object keys reads the typed sub-columns directly (with pruning). + std::shared_ptr variant = Variant(R"({"a": 5, "b": "hi"})"); + std::shared_ptr physical; + std::shared_ptr shredded; + MakeFullShredded( + arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::utf8())}), + variant, &physical, &shredded); + ASSERT_FALSE(HasFatalFailure()); + + auto read_field = arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a"), + AccessChild("b", arrow::utf8(), "$.b")})); + auto file_field = arrow::field("v", physical); + std::shared_ptr plan = CreatePlan(read_field, file_field); + ASSERT_NE(plan, nullptr); + + // With both keys shredded and requested, pruning keeps {metadata, typed_value} and drops the + // top-level `value` column; project the full array down to match the pushed-down physical. + auto made = arrow::StructArray::Make({shredded->field(0), shredded->field(2)}, + {std::string(VariantDefs::kMetadataFieldName), + std::string(VariantDefs::kTypedValueFieldName)}); + ASSERT_TRUE(made.ok()) << made.status().ToString(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled, + plan->Assemble(made.ValueOrDie(), arrow::default_memory_pool())); + auto row = std::static_pointer_cast(assembled); + EXPECT_EQ(static_cast(*row->field(0)).Value(0), 5); + EXPECT_EQ(static_cast(*row->field(1)).GetString(0), "hi"); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, NestedVariantColumnAndTypeMismatch) { + // A struct column holding a variant child: the plan rebuilds the struct around the reassembled + // variant. + std::shared_ptr variant = Variant(R"({"a": 5})"); + std::shared_ptr physical; + std::shared_ptr shredded; + MakeFullShredded(arrow::struct_({arrow::field("a", arrow::int64())}), variant, &physical, + &shredded); + ASSERT_FALSE(HasFatalFailure()); + + auto read_field = arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + VariantTypeUtils::ToArrowField("v")})); + auto file_field = arrow::field( + "s", arrow::struct_({arrow::field("x", arrow::int32()), arrow::field("v", physical)})); + // A sibling plain struct with no variant exercises `ContainsNestedVariant` returning false. + auto plain_field = arrow::field("plain", arrow::struct_({arrow::field("y", arrow::int32())})); + auto plans_result = VariantShreddingReadPlanFactory::CreateReadPlans( + arrow::schema({plain_field, read_field}), arrow::schema({plain_field, file_field}), pool_); + ASSERT_OK_AND_ASSIGN(auto plans, std::move(plans_result)); + ASSERT_EQ(plans.count("plain"), 0); + ASSERT_EQ(plans.count("s"), 1); + std::shared_ptr plan = plans["s"]; + + auto x_array = Int32Array(7); + auto made = arrow::StructArray::Make({x_array, shredded}, std::vector{"x", "v"}); + ASSERT_TRUE(made.ok()) << made.status().ToString(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled, + plan->Assemble(made.ValueOrDie(), arrow::default_memory_pool())); + auto row = std::static_pointer_cast(assembled); + EXPECT_EQ(static_cast(*row->field(0)).Value(0), 7); + ASSERT_EQ(row->field(1)->type_id(), arrow::Type::STRUCT); + + // Assembling a physical array whose type differs from the logical struct is an error. + auto ints = Int32Array(1); + ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool())); +} + +TEST_F(VariantShreddingReadPlanFactoryTest, ColumnAbsentInFileSkipped) { + // Schema evolution: the variant column is missing from the file, so no plan is produced. + auto read_field = VariantTypeUtils::ToArrowField("v"); + auto other = arrow::field("other", arrow::int32()); + ASSERT_OK_AND_ASSIGN(auto plans, + VariantShreddingReadPlanFactory::CreateReadPlans( + arrow::schema({read_field}), arrow::schema({other}), pool_)); + EXPECT_TRUE(plans.empty()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_shredding_test.cpp b/src/paimon/common/data/variant/variant_shredding_test.cpp new file mode 100644 index 00000000..632223a9 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_test.cpp @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_reassembler.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_shredding_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantShreddingTest : public ::testing::Test { + public: + // Shreds the given JSON documents (nullptr = null variant) with the given shredding type, + // asserts the reassembled variants render back to the same JSON, and returns the shredded + // array for structural checks. + std::shared_ptr RoundTrip( + const std::shared_ptr& shredding_type, + const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr physical, + VariantShreddingUtils::VariantShreddingSchema(shredding_type)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + + EXPECT_OK_AND_ASSIGN( + std::unique_ptr writer, + VariantShreddedColumnWriter::Create(schema, physical, arrow::default_memory_pool())); + std::vector expected_jsons; + for (const char* json : jsons) { + if (json == nullptr) { + EXPECT_OK(writer->AppendNull()); + expected_jsons.emplace_back(); + continue; + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::FromJson(json, pool_)); + EXPECT_OK_AND_ASSIGN(std::string expected_json, variant->ToJson()); + expected_jsons.push_back(std::move(expected_json)); + EXPECT_OK(writer->Append(*variant)); + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr shredded_array, writer->Finish()); + auto shredded = std::static_pointer_cast(shredded_array); + + EXPECT_OK_AND_ASSIGN(std::shared_ptr assembled_array, + VariantReassembler::AssembleVariantArray( + shredded, schema, pool_, arrow::default_memory_pool())); + auto assembled = std::static_pointer_cast(assembled_array); + EXPECT_EQ(assembled->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(assembled->field(0)); + auto metadata_column = std::static_pointer_cast(assembled->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + EXPECT_TRUE(assembled->IsNull(i)); + continue; + } + EXPECT_FALSE(assembled->IsNull(i)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + EXPECT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + EXPECT_EQ(actual_json, expected_jsons[i]); + } + return shredded; + } + + protected: + // The physical shredded arrow type for a logical shredding type. + std::shared_ptr Physical(const std::shared_ptr& logical) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr physical, + VariantShreddingUtils::VariantShreddingSchema(logical)); + return physical; + } + + std::shared_ptr Json(const char* json) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::FromJson(json, pool_)); + return variant; + } + + // Builds a single variant using the direct append API, which can encode types (float, binary, + // date, timestamp) that JSON parsing never produces. + std::shared_ptr BuildVariant( + const std::function& append) { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + Status st = append(builder); + EXPECT_TRUE(st.ok()) << st.ToString(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + return variant; + } + + // Shreds the given variants (nullptr = null row) against a physical shredded type, reassembles + // them, asserts each reassembled variant renders back to the same JSON, and returns the + // shredded array. Unlike `RoundTrip`, the physical type is provided directly so that typed + // columns unsupported by `VariantShreddingSchema` (date/timestamp) can be exercised. + std::shared_ptr ShredAndCheck( + const std::shared_ptr& physical, + const std::vector>& variants) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + EXPECT_OK_AND_ASSIGN( + std::unique_ptr writer, + VariantShreddedColumnWriter::Create(schema, physical, arrow::default_memory_pool())); + std::vector expected_jsons; + for (const auto& variant : variants) { + if (variant == nullptr) { + EXPECT_OK(writer->AppendNull()); + expected_jsons.emplace_back(); + continue; + } + EXPECT_OK_AND_ASSIGN(std::string expected_json, variant->ToJson()); + expected_jsons.push_back(std::move(expected_json)); + EXPECT_OK(writer->Append(*variant)); + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr shredded_array, writer->Finish()); + auto shredded = std::static_pointer_cast(shredded_array); + + EXPECT_OK_AND_ASSIGN(std::shared_ptr assembled_array, + VariantReassembler::AssembleVariantArray( + shredded, schema, pool_, arrow::default_memory_pool())); + auto assembled = std::static_pointer_cast(assembled_array); + auto value_column = std::static_pointer_cast(assembled->field(0)); + auto metadata_column = std::static_pointer_cast(assembled->field(1)); + for (size_t i = 0; i < variants.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (variants[i] == nullptr) { + EXPECT_TRUE(assembled->IsNull(i)); + continue; + } + EXPECT_FALSE(assembled->IsNull(i)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + EXPECT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + EXPECT_EQ(actual_json, expected_jsons[i]); + } + return shredded; + } + + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(VariantShreddingTest, ShreddingSchemaShape) { + auto shredding_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr physical, + VariantShreddingUtils::VariantShreddingSchema(shredding_type)); + // struct{metadata: binary not null, value: binary, typed_value: struct{a: + // struct{value, typed_value} not null, b: ... not null}} + auto expected = arrow::struct_( + {arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), true), + arrow::field( + "typed_value", + arrow::struct_( + {arrow::field("a", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", arrow::int32(), true)}), + false), + arrow::field("b", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", arrow::utf8(), true)}), + false)}), + true)}); + ASSERT_TRUE(physical->Equals(*expected)) << physical->ToString(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + ASSERT_EQ(schema->top_level_metadata_idx, 0); + ASSERT_EQ(schema->variant_idx, 1); + ASSERT_EQ(schema->typed_idx, 2); + ASSERT_TRUE(schema->has_object_schema); + ASSERT_EQ(schema->object_schema.size(), 2); + ASSERT_FALSE(schema->IsUnshredded()); + ASSERT_TRUE(VariantShreddingUtils::IsShreddedFileType(physical)); + auto logical_variant = arrow::struct_({arrow::field("value", arrow::binary(), false), + arrow::field("metadata", arrow::binary(), false)}); + auto untyped_physical_variant = + arrow::struct_({arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), false)}); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(logical_variant)); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(untyped_physical_variant)); + ASSERT_FALSE(VariantShreddingUtils::IsUntypedPhysicalVariantType(logical_variant)); + ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(untyped_physical_variant)); + + // Invalid shredding types are rejected. + ASSERT_NOK(VariantShreddingUtils::VariantShreddingSchema(arrow::date32())); + ASSERT_NOK( + VariantShreddingUtils::VariantShreddingSchema(arrow::map(arrow::utf8(), arrow::int32()))); +} + +TEST_F(VariantShreddingTest, ShredObject) { + // Mirrors the Java GenericVariantTest#testShredding scenarios. + auto variant_json = R"({"a": 1, "b": "hello"})"; + // Happy path: all fields shredded, no residual value. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}), + {variant_json}); + ASSERT_TRUE(shredded->field(1)->IsNull(0)); // top-level value (residual) is null + ASSERT_FALSE(shredded->field(2)->IsNull(0)); // typed_value is present + } + // Missing field "c" in the data: present in schema, both children null. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("c", arrow::utf8()), + arrow::field("b", arrow::utf8())}), + {variant_json}); + auto typed = std::static_pointer_cast(shredded->field(2)); + auto c_group = std::static_pointer_cast(typed->field(1)); + ASSERT_FALSE(c_group->IsNull(0)); + ASSERT_TRUE(c_group->field(0)->IsNull(0)); + ASSERT_TRUE(c_group->field(1)->IsNull(0)); + } + // "a" is not present in the shredding schema: it goes to the residual value. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("b", arrow::utf8()), arrow::field("c", arrow::utf8())}), + {variant_json}); + auto value_column = std::static_pointer_cast(shredded->field(1)); + ASSERT_FALSE(value_column->IsNull(0)); + // The residual must equal the standalone encoding of {"a": 1}. + ASSERT_OK_AND_ASSIGN(std::shared_ptr residual_expected, + GenericVariant::FromJson("{\"a\": 1}", pool_)); + ASSERT_OK_AND_ASSIGN(std::string_view residual_value, residual_expected->Value()); + ASSERT_EQ(value_column->GetView(0), residual_value); + } +} + +TEST_F(VariantShreddingTest, ShredAllTypes) { + // Mirrors the Java GenericVariantTest#testShreddingAllTypes. + const char* json = + "{\n" + " \"c1\": \"Hello, World!\",\n" + " \"c2\": 12345678901234,\n" + " \"c3\": 1.0123456789012345678901234567890123456789,\n" + " \"c4\": 100.99,\n" + " \"c5\": true,\n" + " \"c6\": null,\n" + " \"c7\": {\"street\" : \"Main St\",\"city\" : \"Hangzhou\"},\n" + " \"c8\": [1, 2]\n" + "}\n"; + auto shredding_type = arrow::struct_( + {arrow::field("c1", arrow::utf8()), arrow::field("c2", arrow::int64()), + arrow::field("c3", arrow::float64()), arrow::field("c4", arrow::decimal128(5, 2)), + arrow::field("c5", arrow::boolean()), arrow::field("c6", arrow::utf8()), + arrow::field("c7", arrow::struct_({arrow::field("street", arrow::utf8()), + arrow::field("city", arrow::utf8())})), + arrow::field("c8", arrow::list(arrow::int32()))}); + auto shredded = RoundTrip(shredding_type, {json, nullptr, json}); + + // c6 is a variant null: it stays in the field's value column ("00"), typed_value is null. + auto typed = std::static_pointer_cast(shredded->field(2)); + auto c6_group = std::static_pointer_cast(typed->field(5)); + ASSERT_FALSE(c6_group->IsNull(0)); + auto c6_value = std::static_pointer_cast(c6_group->field(0)); + ASSERT_FALSE(c6_value->IsNull(0)); + ASSERT_EQ(c6_value->GetView(0), std::string_view("\x00", 1)); + ASSERT_TRUE(c6_group->field(1)->IsNull(0)); + + // Nothing was left over at the top level. + ASSERT_TRUE(shredded->field(1)->IsNull(0)); + + // No shredding at all: everything stays in the top-level value. + auto no_match_type = arrow::struct_({arrow::field("other", arrow::utf8())}); + auto unshredded = RoundTrip(no_match_type, {json}); + auto value_column = std::static_pointer_cast(unshredded->field(1)); + ASSERT_FALSE(value_column->IsNull(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(json, pool_)); + ASSERT_OK_AND_ASSIGN(std::string_view expected_value, expected->Value()); + ASSERT_EQ(value_column->GetView(0), expected_value); +} + +TEST_F(VariantShreddingTest, ShredScalarsAndMismatches) { + // Top-level scalar shredding with type mismatches falling back to the value column. + auto long_type = arrow::struct_({arrow::field("x", arrow::int64())}); + RoundTrip(long_type, + {"{\"x\": 5}", R"({"x": "not a number"})", "{\"x\": 3.25}", "{\"x\": [1]}", "{}"}); + // Decimal rescale: 100.99 fits decimal(9, 4) exactly (allowNumericScaleChanges). + auto decimal_type = arrow::struct_({arrow::field("x", arrow::decimal128(9, 4))}); + RoundTrip(decimal_type, + {"{\"x\": 100.99}", "{\"x\": 42}", "{\"x\": 0.123456789}", "{\"x\": 100.00}"}); + // A 38-digit unscaled value cannot rescale to scale 1 without overflowing the 128-bit + // decimal; it must fall back to the value column instead of being written corrupted. + auto wide_decimal_type = arrow::struct_({arrow::field("x", arrow::decimal128(38, 1))}); + RoundTrip(wide_decimal_type, + {"{\"x\": 99999999999999999999999999999999999999}", "{\"x\": 1.5}"}); + // Integer target from decimal that is numerically integral. + auto int_type = arrow::struct_({arrow::field("x", arrow::int32())}); + RoundTrip(int_type, {"{\"x\": 5.0}", "{\"x\": 5.5}", "{\"x\": 123456789012345678}"}); + auto array_type = arrow::struct_( + {arrow::field("arr", arrow::list(arrow::struct_({arrow::field("k", arrow::utf8())})))}); + RoundTrip(array_type, {R"({"arr": [{"k": "v1"}, {"k": "v2", "extra": 1}, {"other": 2}]})", + "{\"arr\": [1, 2]}", R"({"arr": {"k": "v"}})"}); +} + +TEST_F(VariantShreddingTest, ScalarShreddingIntegerWidths) { + // int8/int16 targets: in-range longs shred into the narrow typed column, while out-of-range + // values fall back to the residual value column. Decimal-integral inputs take the + // decimal->integer shredding path. + ShredAndCheck(Physical(arrow::struct_({arrow::field("x", arrow::int8())})), + {Json(R"({"x": 5})"), Json(R"({"x": 200})"), Json(R"({"x": 5.0})")}); + ShredAndCheck(Physical(arrow::struct_({arrow::field("x", arrow::int16())})), + {Json(R"({"x": 5})"), Json(R"({"x": 40000})"), Json(R"({"x": 5.0})")}); +} + +TEST_F(VariantShreddingTest, ScalarShreddingFloatAndBinary) { + // Float and binary variants are not producible from JSON, so build them directly. They shred + // into their typed columns and round-trip back to the same value. + ShredAndCheck(Physical(arrow::float32()), + {BuildVariant([](VariantBuilder& b) { return b.AppendFloat(1.5f); }), nullptr}); + ShredAndCheck(Physical(arrow::binary()), {BuildVariant([](VariantBuilder& b) { + return b.AppendBinary(std::string_view("\x01\x02\x03", 3)); + })}); +} + +TEST_F(VariantShreddingTest, ScalarShreddingDate) { + // date typed columns are produced by external engines; `VariantShreddingSchema` itself never + // emits them, so build the physical shredded type directly. + auto physical = arrow::struct_({arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", arrow::date32(), true)}); + ShredAndCheck(physical, + {BuildVariant([](VariantBuilder& b) { return b.AppendDate(19000); }), nullptr}); +} + +TEST_F(VariantShreddingTest, TimestampSchemaParsing) { + auto make_physical = [](const std::shared_ptr& ts) { + return arrow::struct_({arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", ts, true)}); + }; + // A microsecond timestamp with a timezone parses as TIMESTAMP_LTZ; without, TIMESTAMP_NTZ. + ASSERT_OK_AND_ASSIGN(std::shared_ptr ltz, + VariantShreddingUtils::BuildVariantSchema( + make_physical(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")))); + ASSERT_TRUE(ltz->scalar_schema.has_value()); + ASSERT_EQ(ltz->scalar_schema->kind, VariantSchema::ScalarKind::kTimestampLtz); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ntz, + VariantShreddingUtils::BuildVariantSchema( + make_physical(arrow::timestamp(arrow::TimeUnit::MICRO)))); + ASSERT_TRUE(ntz->scalar_schema.has_value()); + ASSERT_EQ(ntz->scalar_schema->kind, VariantSchema::ScalarKind::kTimestampNtz); + // Non-microsecond timestamps cannot represent the variant's microsecond values, so they are + // rejected as an invalid shredding schema. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema( + make_physical(arrow::timestamp(arrow::TimeUnit::MILLI, "UTC")))); +} + +TEST_F(VariantShreddingTest, TimestampReassembly) { + // The writer never produces timestamp typed columns, but the reassembler must handle files + // written by engines that do. Build a shredded array with a populated timestamp typed_value + // and verify it reassembles into the same variant. + auto reassemble_one = [&](const std::shared_ptr& ts_type, + const std::shared_ptr& reference, int64_t micros) { + auto physical = arrow::struct_({arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", ts_type, true)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + std::string metadata(reference->Metadata()); + ASSERT_OK_AND_ASSIGN(std::string expected_json, reference->ToJson()); + + arrow::BinaryBuilder meta_builder; + ASSERT_TRUE(meta_builder.Append(metadata).ok()); + std::shared_ptr meta_array; + ASSERT_TRUE(meta_builder.Finish(&meta_array).ok()); + arrow::BinaryBuilder value_builder; + ASSERT_TRUE(value_builder.AppendNull().ok()); + std::shared_ptr value_array; + ASSERT_TRUE(value_builder.Finish(&value_array).ok()); + arrow::TimestampBuilder ts_builder(ts_type, arrow::default_memory_pool()); + ASSERT_TRUE(ts_builder.Append(micros).ok()); + std::shared_ptr ts_array; + ASSERT_TRUE(ts_builder.Finish(&ts_array).ok()); + auto made = arrow::StructArray::Make({meta_array, value_array, ts_array}, + {"metadata", "value", "typed_value"}); + ASSERT_TRUE(made.ok()) << made.status().ToString(); + std::shared_ptr shredded = made.ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled_array, + VariantReassembler::AssembleVariantArray( + shredded, schema, pool_, arrow::default_memory_pool())); + auto assembled = std::static_pointer_cast(assembled_array); + auto value_column = std::static_pointer_cast(assembled->field(0)); + auto metadata_column = std::static_pointer_cast(assembled->field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_EQ(actual_json, expected_json); + }; + + int64_t micros = 1700000000000000; + reassemble_one(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + BuildVariant([&](VariantBuilder& b) { return b.AppendTimestamp(micros); }), + micros); + reassemble_one(arrow::timestamp(arrow::TimeUnit::MICRO), + BuildVariant([&](VariantBuilder& b) { return b.AppendTimestampNtz(micros); }), + micros); +} + +TEST_F(VariantShreddingTest, ScalarSchemaToArrowType) { + using SK = VariantSchema::ScalarKind; + auto check = [](VariantSchema::ScalarType scalar, + const std::shared_ptr& expected) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr type, + VariantShreddingUtils::ScalarSchemaToArrowType(scalar)); + ASSERT_TRUE(type->Equals(*expected)) << type->ToString(); + }; + check({SK::kBoolean}, arrow::boolean()); + check({SK::kByte}, arrow::int8()); + check({SK::kShort}, arrow::int16()); + check({SK::kInt}, arrow::int32()); + check({SK::kLong}, arrow::int64()); + check({SK::kFloat}, arrow::float32()); + check({SK::kDouble}, arrow::float64()); + check({SK::kString}, arrow::utf8()); + check({SK::kBinary}, arrow::binary()); + check({SK::kDecimal, 10, 2}, arrow::decimal128(10, 2)); + check({SK::kDate}, arrow::date32()); + check({SK::kTimestampLtz}, arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")); + check({SK::kTimestampNtz}, arrow::timestamp(arrow::TimeUnit::MICRO)); + // kUuid has no shredded arrow representation. + ASSERT_NOK(VariantShreddingUtils::ScalarSchemaToArrowType({SK::kUuid})); +} + +TEST_F(VariantShreddingTest, InvalidShreddingSchemas) { + // Not a struct. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::int32())); + // Empty struct. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::struct_({}))); + // The "value" column must be binary. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema( + arrow::struct_({arrow::field("value", arrow::int32())}))); + // Unknown field name. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema( + arrow::struct_({arrow::field("bogus", arrow::binary())}))); + // A top-level schema must carry a metadata column. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema( + arrow::struct_({arrow::field("value", arrow::binary())}))); + // Unsupported typed_value type. + ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::struct_( + {arrow::field("metadata", arrow::binary()), arrow::field("value", arrow::binary()), + arrow::field("typed_value", arrow::map(arrow::utf8(), arrow::int32()))}))); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_shredding_utils.cpp b/src/paimon/common/data/variant/variant_shredding_utils.cpp new file mode 100644 index 00000000..bbb23135 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_utils.cpp @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_shredding_utils.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_type_utils.h" + +namespace paimon { + +namespace { + +Status InvalidVariantShreddingSchema(const std::shared_ptr& type) { + return Status::Invalid( + fmt::format("Invalid variant shredding schema: {}", type ? type->ToString() : "null")); +} + +// Mirrors the Java `PaimonShreddingUtils.variantShreddingSchema(dataType, isTopLevel, +// isObjectField)`. +Result> VariantShreddingSchemaImpl( + const std::shared_ptr& data_type, bool is_top_level, bool is_object_field) { + arrow::FieldVector fields; + if (is_top_level) { + fields.push_back(arrow::field(VariantDefs::kMetadataFieldName, arrow::binary(), + /*nullable=*/false)); + } + switch (data_type->id()) { + case arrow::Type::LIST: { + const auto& list_type = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_type, + VariantShreddingSchemaImpl(list_type->value_type(), + /*is_top_level=*/false, + /*is_object_field=*/false)); + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, + arrow::list(element_type), /*nullable=*/true)); + break; + } + case arrow::Type::STRUCT: { + // The field name level is always non-nullable: Variant null values are represented in + // the "value" column as "00", and missing values are represented by setting both + // "value" and "typed_value" to null. + const auto& struct_type = std::static_pointer_cast(data_type); + arrow::FieldVector shredded_fields; + for (const auto& field : struct_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_type, + VariantShreddingSchemaImpl(field->type(), + /*is_top_level=*/false, + /*is_object_field=*/true)); + shredded_fields.push_back( + arrow::field(field->name(), field_type, /*nullable=*/false)); + } + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, + arrow::struct_(shredded_fields), /*nullable=*/true)); + break; + } + case arrow::Type::NA: { + // `arrow::null()` denotes an untyped VARIANT leaf in shredding types. It doesn't + // need a typed column. If there is no typed column, value is required for array + // elements or top-level fields, but optional for objects (where a null represents a + // missing field). + fields.push_back(arrow::field(VariantDefs::kValueFieldName, arrow::binary(), + /*nullable=*/is_object_field)); + break; + } + case arrow::Type::STRING: + case arrow::Type::BOOL: + case arrow::Type::BINARY: + case arrow::Type::DECIMAL128: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: { + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, data_type, + /*nullable=*/true)); + break; + } + default: + return InvalidVariantShreddingSchema(data_type); + } + return arrow::struct_(fields); +} + +Result> BuildVariantSchemaImpl( + const std::shared_ptr& type, bool top_level) { + if (type->id() != arrow::Type::STRUCT) { + return InvalidVariantShreddingSchema(type); + } + const auto& struct_type = std::static_pointer_cast(type); + // The struct must not be empty or contain duplicate field names. The latter is enforced in + // the loop below. + if (struct_type->num_fields() == 0) { + return InvalidVariantShreddingSchema(type); + } + + auto schema = std::make_shared(); + schema->num_fields = struct_type->num_fields(); + + for (int32_t i = 0; i < struct_type->num_fields(); ++i) { + const auto& field = struct_type->field(i); + const auto& field_type = field->type(); + if (field->name() == VariantDefs::kTypedValueFieldName) { + if (schema->typed_idx != -1) { + return InvalidVariantShreddingSchema(type); + } + schema->typed_idx = i; + switch (field_type->id()) { + case arrow::Type::STRUCT: { + const auto& object_type = + std::static_pointer_cast(field_type); + schema->has_object_schema = true; + schema->object_schema.reserve(object_type->num_fields()); + for (int32_t index = 0; index < object_type->num_fields(); ++index) { + const auto& object_field = object_type->field(index); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr field_schema, + BuildVariantSchemaImpl(object_field->type(), /*top_level=*/false)); + schema->object_schema.push_back( + VariantSchema::ObjectField{object_field->name(), field_schema}); + auto [it, inserted] = + schema->object_schema_map.emplace(object_field->name(), index); + if (!inserted) { + return InvalidVariantShreddingSchema(type); + } + } + break; + } + case arrow::Type::LIST: { + const auto& list_type = std::static_pointer_cast(field_type); + PAIMON_ASSIGN_OR_RAISE( + schema->array_schema, + BuildVariantSchemaImpl(list_type->value_type(), /*top_level=*/false)); + break; + } + case arrow::Type::BOOL: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kBoolean}; + break; + case arrow::Type::INT8: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kByte}; + break; + case arrow::Type::INT16: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kShort}; + break; + case arrow::Type::INT32: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kInt}; + break; + case arrow::Type::INT64: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kLong}; + break; + case arrow::Type::FLOAT: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kFloat}; + break; + case arrow::Type::DOUBLE: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDouble}; + break; + case arrow::Type::STRING: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kString}; + break; + case arrow::Type::BINARY: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kBinary}; + break; + case arrow::Type::DATE32: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDate}; + break; + case arrow::Type::DECIMAL128: { + const auto& decimal_type = + std::static_pointer_cast(field_type); + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDecimal, + decimal_type->precision(), decimal_type->scale()}; + break; + } + case arrow::Type::TIMESTAMP: { + const auto& timestamp_type = + std::static_pointer_cast(field_type); + // The variant binary stores timestamps as microseconds since the epoch; a + // typed_value column of any other precision would misinterpret the values. + if (timestamp_type->unit() != arrow::TimeUnit::MICRO) { + return InvalidVariantShreddingSchema(type); + } + schema->scalar_schema = + VariantSchema::ScalarType{timestamp_type->timezone().empty() + ? VariantSchema::ScalarKind::kTimestampNtz + : VariantSchema::ScalarKind::kTimestampLtz}; + break; + } + default: + return InvalidVariantShreddingSchema(type); + } + } else if (field->name() == VariantDefs::kValueFieldName) { + if (schema->variant_idx != -1 || field_type->id() != arrow::Type::BINARY) { + return InvalidVariantShreddingSchema(type); + } + schema->variant_idx = i; + } else if (field->name() == VariantDefs::kMetadataFieldName) { + if (schema->top_level_metadata_idx != -1 || field_type->id() != arrow::Type::BINARY) { + return InvalidVariantShreddingSchema(type); + } + schema->top_level_metadata_idx = i; + } else { + return InvalidVariantShreddingSchema(type); + } + } + + if (top_level != (schema->top_level_metadata_idx >= 0)) { + return InvalidVariantShreddingSchema(type); + } + return schema; +} + +} // namespace + +Result> VariantShreddingUtils::VariantShreddingSchema( + const std::shared_ptr& shredding_type) { + return VariantShreddingSchemaImpl(shredding_type, /*is_top_level=*/true, + /*is_object_field=*/false); +} + +Result> VariantShreddingUtils::BuildVariantSchema( + const std::shared_ptr& struct_type) { + return BuildVariantSchemaImpl(struct_type, /*top_level=*/true); +} + +Result> VariantShreddingUtils::ScalarSchemaToArrowType( + const VariantSchema::ScalarType& scalar) { + switch (scalar.kind) { + case VariantSchema::ScalarKind::kBoolean: + return arrow::boolean(); + case VariantSchema::ScalarKind::kByte: + return arrow::int8(); + case VariantSchema::ScalarKind::kShort: + return arrow::int16(); + case VariantSchema::ScalarKind::kInt: + return arrow::int32(); + case VariantSchema::ScalarKind::kLong: + return arrow::int64(); + case VariantSchema::ScalarKind::kFloat: + return arrow::float32(); + case VariantSchema::ScalarKind::kDouble: + return arrow::float64(); + case VariantSchema::ScalarKind::kString: + return arrow::utf8(); + case VariantSchema::ScalarKind::kBinary: + return arrow::binary(); + case VariantSchema::ScalarKind::kDecimal: + return arrow::decimal128(scalar.precision, scalar.scale); + case VariantSchema::ScalarKind::kDate: + return arrow::date32(); + case VariantSchema::ScalarKind::kTimestampLtz: + return arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + case VariantSchema::ScalarKind::kTimestampNtz: + return arrow::timestamp(arrow::TimeUnit::MICRO); + default: + return Status::NotImplemented(fmt::format("Unsupported variant scalar kind: {}", + static_cast(scalar.kind))); + } +} + +bool VariantShreddingUtils::IsShreddedFileType( + const std::shared_ptr& file_variant_type) { + if (!file_variant_type || file_variant_type->id() != arrow::Type::STRUCT) { + return false; + } + const auto& struct_type = std::static_pointer_cast(file_variant_type); + return struct_type->GetFieldByName(VariantDefs::kTypedValueFieldName) != nullptr; +} + +bool VariantShreddingUtils::IsUntypedPhysicalVariantType( + const std::shared_ptr& file_variant_type) { + if (!file_variant_type || file_variant_type->id() != arrow::Type::STRUCT) { + return false; + } + const auto& struct_type = std::static_pointer_cast(file_variant_type); + if (struct_type->num_fields() != 2) { + return false; + } + const auto& metadata = struct_type->field(0); + const auto& value = struct_type->field(1); + return metadata->name() == VariantDefs::kMetadataFieldName && + metadata->type()->id() == arrow::Type::BINARY && + value->name() == VariantDefs::kValueFieldName && + value->type()->id() == arrow::Type::BINARY; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_utils.h b/src/paimon/common/data/variant/variant_shredding_utils.h new file mode 100644 index 00000000..7aa59e1d --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_utils.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +} // namespace arrow + +namespace paimon { + +/// Utils for converting between shredding schemas (`VariantSchema`) and their physical Arrow +/// representation, mirroring the Java `PaimonShreddingUtils` schema functions. +class VariantShreddingUtils { + public: + VariantShreddingUtils() = delete; + ~VariantShreddingUtils() = delete; + + /// Given an expected schema of a Variant value, returns a suitable physical schema for + /// shredding, by inserting appropriate intermediate value/typed_value fields at each level. + /// For example, to represent the JSON `{"a": 1, "b": "hello"}`, the schema + /// `struct{a: int32, b: string}` could be passed into this function, and it would return the + /// shredding schema: `struct{metadata: binary, value: binary, typed_value: struct{a: + /// struct{value: binary, typed_value: int32}, b: struct{value: binary, typed_value: + /// string}}}`. + static Result> VariantShreddingSchema( + const std::shared_ptr& shredding_type); + + /// Builds a `VariantSchema` from the physical shredded struct type (the inverse of + /// `VariantShreddingSchema`), validating field names and types. + static Result> BuildVariantSchema( + const std::shared_ptr& struct_type); + + /// The Arrow type of the typed_value column for a scalar shredding schema. + static Result> ScalarSchemaToArrowType( + const VariantSchema::ScalarType& scalar); + + /// Whether the physical struct type of a variant field in a data file is shredded (contains + /// a `typed_value` child). + static bool IsShreddedFileType(const std::shared_ptr& file_variant_type); + + /// Whether the physical struct uses the untyped inference layout `{metadata, value}`. + static bool IsUntypedPhysicalVariantType( + const std::shared_ptr& file_variant_type); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp new file mode 100644 index 00000000..7fccc4ff --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan.h" + +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_type_json_parser.h" +#include "rapidjson/document.h" + +namespace paimon { + +namespace { + +/// Recursively rebuilds `field`, replacing the variant fields planned under `paths` (grouped by +/// their leading index at this level) with their shredded physical types. +Result> ReplacePlannedFields( + const std::shared_ptr& field, + const std::map, std::shared_ptr>& paths, size_t depth, + std::vector* columns) { + const auto& struct_type = + arrow::internal::checked_cast(*field->type()); + arrow::FieldVector new_fields = struct_type.fields(); + bool changed = false; + auto it = paths.begin(); + while (it != paths.end()) { + int32_t index = it->first[depth]; + // Collect the consecutive paths that descend into the same child. + std::map, std::shared_ptr> child_paths; + for (; it != paths.end() && it->first[depth] == index; ++it) { + child_paths.emplace(it->first, it->second); + } + if (index < 0 || index >= struct_type.num_fields()) { + return Status::Invalid( + fmt::format("variant shredding path index {} is out of bounds", index)); + } + const std::shared_ptr& child = struct_type.field(index); + auto terminal = child_paths.begin(); + if (terminal->first.size() == depth + 1) { + // The path terminates at this child: it must be a variant field. + if (child_paths.size() > 1 || !VariantTypeUtils::IsVariantField(child)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_type, + VariantShreddingUtils::VariantShreddingSchema(terminal->second)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant_schema, + VariantShreddingUtils::BuildVariantSchema(physical_type)); + columns->push_back(VariantShreddingWritePlan::PlannedColumn{ + terminal->first, std::move(variant_schema), physical_type}); + new_fields[index] = child->WithType(physical_type); + changed = true; + } else { + // The paths descend into a nested struct child. + if (child->type()->id() != arrow::Type::STRUCT || + VariantTypeUtils::IsVariantField(child)) { + continue; + } + size_t planned_before = columns->size(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_child, + ReplacePlannedFields(child, child_paths, depth + 1, columns)); + if (columns->size() > planned_before) { + new_fields[index] = new_child; + changed = true; + } + } + } + if (!changed) { + return field; + } + return field->WithType(arrow::struct_(new_fields)); +} + +Status CollectPlannedColumns(const std::shared_ptr& logical_field, + const std::shared_ptr& physical_field, + std::vector* path, + std::vector* columns) { + if (logical_field->name() != physical_field->name()) { + return Status::Invalid( + fmt::format("variant shredding physical field '{}' does not match logical field '{}'", + physical_field->name(), logical_field->name())); + } + if (VariantTypeUtils::IsVariantField(logical_field)) { + if (logical_field->type()->Equals(*physical_field->type())) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant_schema, + VariantShreddingUtils::BuildVariantSchema(physical_field->type())); + columns->push_back(VariantShreddingWritePlan::PlannedColumn{ + *path, std::move(variant_schema), physical_field->type()}); + return Status::OK(); + } + if (logical_field->type()->Equals(*physical_field->type())) { + return Status::OK(); + } + if (logical_field->type()->id() != arrow::Type::STRUCT || + physical_field->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format( + "variant shredding physical type of field '{}' differs outside a Variant column", + logical_field->name())); + } + const auto& logical_type = + arrow::internal::checked_cast(*logical_field->type()); + const auto& physical_type = + arrow::internal::checked_cast(*physical_field->type()); + if (logical_type.num_fields() != physical_type.num_fields()) { + return Status::Invalid( + fmt::format("variant shredding physical struct '{}' has a different field count", + logical_field->name())); + } + for (int32_t i = 0; i < logical_type.num_fields(); ++i) { + path->push_back(i); + PAIMON_RETURN_NOT_OK( + CollectPlannedColumns(logical_type.field(i), physical_type.field(i), path, columns)); + path->pop_back(); + } + return Status::OK(); +} + +} // namespace + +Result> VariantShreddingWritePlan::Create( + const std::shared_ptr& logical_schema, + const std::map>& column_shredding_types) { + std::map, std::shared_ptr> path_shredding_types; + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + auto it = column_shredding_types.find(logical_schema->field(i)->name()); + if (it != column_shredding_types.end()) { + path_shredding_types.emplace(std::vector{i}, it->second); + } + } + return CreateFromPaths(logical_schema, path_shredding_types); +} + +Result> VariantShreddingWritePlan::CreateFromPaths( + const std::shared_ptr& logical_schema, + const std::map, std::shared_ptr>& path_shredding_types) { + std::vector columns; + auto root_field = arrow::field("root", arrow::struct_(logical_schema->fields())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr new_root, + ReplacePlannedFields(root_field, path_shredding_types, /*depth=*/0, &columns)); + auto physical_schema = arrow::schema(new_root->type()->fields(), logical_schema->metadata()); + return std::shared_ptr(new VariantShreddingWritePlan( + logical_schema, std::move(physical_schema), std::move(columns))); +} + +Result> +VariantShreddingWritePlan::CreateFromPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema) { + if (logical_schema->num_fields() != physical_schema->num_fields()) { + return Status::Invalid( + "variant shredding logical and physical schemas have different field counts"); + } + std::vector columns; + std::vector path; + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + path.push_back(i); + PAIMON_RETURN_NOT_OK(CollectPlannedColumns(logical_schema->field(i), + physical_schema->field(i), &path, &columns)); + path.pop_back(); + } + return std::shared_ptr( + new VariantShreddingWritePlan(logical_schema, physical_schema, std::move(columns))); +} + +Result> VariantShreddingWritePlan::FromConfiguredSchema( + const std::shared_ptr& logical_schema, + const std::string& configured_schema_json) { + rapidjson::Document doc; + doc.Parse(configured_schema_json.c_str()); + if (doc.HasParseError()) { + return Status::Invalid(fmt::format("failed to parse variant shredding schema json: {}", + configured_schema_json)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr configured_field, + DataTypeJsonParser::ParseType("shredding_schema", doc)); + if (configured_field->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid("variant shredding schema must be a ROW type"); + } + std::map> column_shredding_types; + for (const auto& column_field : configured_field->type()->fields()) { + column_shredding_types.emplace(column_field->name(), column_field->type()); + } + return Create(logical_schema, column_shredding_types); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.h b/src/paimon/common/data/variant/variant_shredding_write_plan.h new file mode 100644 index 00000000..2fa7a886 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan.h @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class Schema; +} // namespace arrow + +namespace paimon { + +/// A physical write plan for variant shredding: maps the logical write schema (with variant +/// columns) to the physical schema where planned variant columns are replaced by their shredded +/// struct representation. Variant columns may be at the top level or nested inside ROW (struct) +/// columns; variants nested inside arrays or maps are never shredded (as in Java). +class VariantShreddingWritePlan { + public: + /// One planned variant column, identified by its field-index path from the schema root, + /// descending only through struct fields (e.g. `{1, 2}` is the third field of the second + /// top-level column). + struct PlannedColumn { + std::vector path; + std::shared_ptr variant_schema; + std::shared_ptr physical_type; + }; + + /// Creates a plan shredding the given top-level variant columns. + /// + /// @param logical_schema The logical write schema. + /// @param column_shredding_types The shredding type per variant column name, e.g. + /// `{"v": struct{a: int32, b: string}}`. Names that are not top-level variant columns + /// of `logical_schema` are ignored. When no name matches, returns an identity plan. + static Result> Create( + const std::shared_ptr& logical_schema, + const std::map>& column_shredding_types); + + /// Creates a plan shredding the variant columns at the given field-index paths (top-level or + /// nested inside structs). Paths that do not point at a variant field are ignored. When no + /// path matches, returns an identity plan. + static Result> CreateFromPaths( + const std::shared_ptr& logical_schema, + const std::map, std::shared_ptr>& + path_shredding_types); + + /// Creates a plan by comparing the complete logical and physical row schemas produced by + /// whole-row Variant schema inference. + static Result> CreateFromPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema); + + /// Creates a plan from the `variant.shreddingSchema` option value: a ROW type JSON whose + /// fields map top-level variant column names to their shredding types (nested variant + /// columns cannot be configured, as in Java). + static Result> FromConfiguredSchema( + const std::shared_ptr& logical_schema, + const std::string& configured_schema_json); + + const std::shared_ptr& LogicalSchema() const { + return logical_schema_; + } + + const std::shared_ptr& PhysicalSchema() const { + return physical_schema_; + } + + /// The planned variant columns, ordered by path. + const std::vector& Columns() const { + return columns_; + } + + private: + VariantShreddingWritePlan(std::shared_ptr logical_schema, + std::shared_ptr physical_schema, + std::vector columns) + : logical_schema_(std::move(logical_schema)), + physical_schema_(std::move(physical_schema)), + columns_(std::move(columns)) {} + + std::shared_ptr logical_schema_; + std::shared_ptr physical_schema_; + std::vector columns_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp new file mode 100644 index 00000000..af7b73f0 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" + +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/common/data/variant/variant_type_utils.h" + +namespace paimon { + +namespace { + +bool ContainsVariantFields(const arrow::FieldVector& fields) { + for (const std::shared_ptr& field : fields) { + if (VariantTypeUtils::IsVariantField(field)) { + return true; + } + if (field->type()->id() == arrow::Type::STRUCT && + ContainsVariantFields(field->type()->fields())) { + return true; + } + } + return false; +} + +} // namespace + +VariantShreddingWritePlanFactory::VariantShreddingWritePlanFactory( + std::optional configured_schema, bool infer_enabled, int32_t max_schema_width, + int32_t max_schema_depth, double min_field_cardinality_ratio, int32_t max_infer_buffer_row, + VariantShreddingInferenceMode inference_mode, int32_t adaptive_max_infer_buffer_row, + double adaptive_retention_ratio, const std::shared_ptr& write_schema, + const std::shared_ptr& pool) + : write_schema_(write_schema), + pool_(pool), + configured_schema_(std::move(configured_schema)), + infer_enabled_(infer_enabled), + max_schema_width_(max_schema_width), + max_schema_depth_(max_schema_depth), + min_field_cardinality_ratio_(min_field_cardinality_ratio), + max_infer_buffer_row_(max_infer_buffer_row), + adaptive_max_infer_buffer_row_(adaptive_max_infer_buffer_row) { + if (!configured_schema_.has_value() && infer_enabled_ && + inference_mode == VariantShreddingInferenceMode::ADAPTIVE) { + adaptive_session_ = std::make_unique( + InferVariantShreddingSchema(write_schema_, pool_, max_schema_width_, max_schema_depth_, + min_field_cardinality_ratio_), + adaptive_max_infer_buffer_row_, min_field_cardinality_ratio_, adaptive_retention_ratio); + } +} + +std::shared_ptr VariantShreddingWritePlanFactory::Create( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool) { + return std::shared_ptr(new VariantShreddingWritePlanFactory( + options.GetVariantShreddingSchema(), options.VariantInferShreddingSchemaEnabled(), + options.GetVariantShreddingMaxSchemaWidth(), options.GetVariantShreddingMaxSchemaDepth(), + options.GetVariantShreddingMinFieldCardinalityRatio(), + options.GetVariantShreddingMaxInferBufferRow(), options.GetVariantShreddingInferenceMode(), + options.GetVariantShreddingAdaptiveMaxInferBufferRow(), + options.GetVariantShreddingAdaptiveRetentionRatio(), write_schema, pool)); +} + +bool VariantShreddingWritePlanFactory::ShouldCreateWritePlan() const { + return ContainsShreddableVariantField() && (HasConfiguredShreddingSchema() || infer_enabled_); +} + +bool VariantShreddingWritePlanFactory::ShouldInferWritePlan() const { + return ContainsShreddableVariantField() && !HasConfiguredShreddingSchema() && infer_enabled_; +} + +int32_t VariantShreddingWritePlanFactory::InferBufferRowCount() const { + if (adaptive_session_ != nullptr && adaptive_session_->HasPrior()) { + return adaptive_max_infer_buffer_row_; + } + return max_infer_buffer_row_; +} + +bool VariantShreddingWritePlanFactory::HasConfiguredShreddingSchema() const { + return configured_schema_.has_value(); +} + +bool VariantShreddingWritePlanFactory::ContainsShreddableVariantField() const { + return ContainsVariantFields(write_schema_->fields()); +} + +Result> VariantShreddingWritePlanFactory::CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) { + if (file_format_identifier != "parquet") { + return Status::NotImplemented( + fmt::format("variant shredding is only supported by the parquet file format, got {}", + file_format_identifier)); + } + + std::shared_ptr plan; + if (HasConfiguredShreddingSchema()) { + PAIMON_ASSIGN_OR_RAISE(plan, VariantShreddingWritePlan::FromConfiguredSchema( + write_schema_, configured_schema_.value())); + } else { + std::shared_ptr physical_schema; + if (adaptive_session_ != nullptr) { + PAIMON_ASSIGN_OR_RAISE(physical_schema, adaptive_session_->InferSchema(sample_batches)); + has_pending_adaptive_inference_ = true; + } else { + InferVariantShreddingSchema inferrer(write_schema_, pool_, max_schema_width_, + max_schema_depth_, min_field_cardinality_ratio_); + PAIMON_ASSIGN_OR_RAISE(physical_schema, inferrer.InferSchema(sample_batches)); + } + PAIMON_ASSIGN_OR_RAISE(plan, VariantShreddingWritePlan::CreateFromPhysicalSchema( + write_schema_, physical_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + std::shared_ptr result = std::move(converter); + if (adaptive_session_ != nullptr) { + pending_adaptive_converter_ = result; + } + return result; +} + +Status VariantShreddingWritePlanFactory::OnFileCompleted( + const std::shared_ptr& converter) { + if (adaptive_session_ == nullptr) { + return Status::OK(); + } + if (!has_pending_adaptive_inference_ || converter != pending_adaptive_converter_) { + return Status::Invalid( + "Completed Variant write plan does not match the pending adaptive inference."); + } + PAIMON_RETURN_NOT_OK(adaptive_session_->CommitPendingInference()); + has_pending_adaptive_inference_ = false; + pending_adaptive_converter_.reset(); + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h new file mode 100644 index 00000000..69b0fa79 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_shredding_inference_session.h" +#include "paimon/core/core_options.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Creates VARIANT shredding batch converters, either from the configured +/// `variant.shreddingSchema` or, when `variant.inferShreddingSchema` is enabled, by inferring a +/// shredding schema from sampled rows buffered per file. Variant columns nested inside ROW +/// (struct) columns are inferred and shredded too; variants inside arrays or maps are not (as in +/// Java). +class VariantShreddingWritePlanFactory : public ShreddingWritePlanFactory { + public: + /// Creates the factory. The `variant.*` option values are already parsed and validated by + /// `CoreOptions::FromMap`, so creation cannot fail on configuration. + static std::shared_ptr Create( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool); + + bool ShouldCreateWritePlan() const override; + + bool ShouldInferWritePlan() const override; + + int32_t InferBufferRowCount() const override; + + Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) override; + + MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression) const override { + // The shredded physical schema is self-describing; no per-file metadata is needed. + return nullptr; + } + + Status OnFileCompleted(const std::shared_ptr& converter) override; + + private: + VariantShreddingWritePlanFactory( + std::optional configured_schema, bool infer_enabled, int32_t max_schema_width, + int32_t max_schema_depth, double min_field_cardinality_ratio, int32_t max_infer_buffer_row, + VariantShreddingInferenceMode inference_mode, int32_t adaptive_max_infer_buffer_row, + double adaptive_retention_ratio, const std::shared_ptr& write_schema, + const std::shared_ptr& pool); + + bool HasConfiguredShreddingSchema() const; + /// Whether the write schema holds a shreddable variant field: at the top level or nested + /// inside structs only (variants inside arrays or maps are never shredded). + bool ContainsShreddableVariantField() const; + + std::shared_ptr write_schema_; + std::shared_ptr pool_; + + std::optional configured_schema_; + bool infer_enabled_ = false; + int32_t max_schema_width_ = 0; + int32_t max_schema_depth_ = 0; + double min_field_cardinality_ratio_ = 0.0; + int32_t max_infer_buffer_row_ = 0; + int32_t adaptive_max_infer_buffer_row_ = 0; + std::unique_ptr adaptive_session_; + bool has_pending_adaptive_inference_ = false; + std::shared_ptr pending_adaptive_converter_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp new file mode 100644 index 00000000..a3d3c6c5 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp @@ -0,0 +1,598 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +class VariantShreddingWritePlanFactoryTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + } + + Result MakeOptions(std::map options) const { + // Keep the manifest format resolvable in test binaries without the avro plugin. + options.emplace("manifest.format", "parquet"); + return CoreOptions::FromMap(options); + } + + std::shared_ptr BuildBatch(const std::vector& jsons) { + auto result = + VariantTestData::BuildVariantBatch(schema_->field(0), schema_->field(1), jsons, pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return std::move(result).value(); + } + + static std::shared_ptr ShreddedTypedValue( + const arrow::StructType& typed_object, const std::string& field_name) { + auto field = typed_object.GetFieldByName(field_name); + if (field == nullptr || field->type()->id() != arrow::Type::STRUCT) { + return nullptr; + } + return std::static_pointer_cast(field->type()) + ->GetFieldByName("typed_value") + ->type(); + } + + protected: + std::shared_ptr pool_; + std::shared_ptr schema_; +}; + +TEST_F(VariantShreddingWritePlanFactoryTest, InactiveWithoutOptions) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeOptions({})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_FALSE(factory->ShouldCreateWritePlan()); + ASSERT_FALSE(factory->ShouldInferWritePlan()); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, ConfiguredSchema) { + const char* shredding_schema_json = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ + {"id": 1, "name": "age", "type": "INT"}, + {"id": 2, "name": "city", "type": "STRING"} + ] + } + } ] + })"; + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.shreddingSchema", shredding_schema_json}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_FALSE(factory->ShouldInferWritePlan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_NE(converter, nullptr); + auto variant_field = converter->GetPhysicalSchema()->GetFieldByName("v"); + ASSERT_NE(variant_field, nullptr); + const auto& physical_type = static_cast(*variant_field->type()); + ASSERT_NE(physical_type.GetFieldByName("typed_value"), nullptr); + // Variant shredding only supports the parquet format. + ASSERT_TRUE(factory->CreateConverter("orc", {}).status().IsNotImplemented()); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchema) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + ASSERT_EQ(factory->InferBufferRowCount(), 4096); + + std::vector> samples = { + BuildBatch({R"({"age": 35, "city": "Chicago"})", R"({"age": 25, "city": "Hangzhou"})"}), + BuildBatch({R"({"age": 18, "city": "Beijing"})", nullptr})}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + auto variant_field = converter->GetPhysicalSchema()->GetFieldByName("v"); + ASSERT_NE(variant_field, nullptr); + const auto& physical_type = static_cast(*variant_field->type()); + auto typed_value = physical_type.GetFieldByName("typed_value"); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + ASSERT_NE(typed_struct.GetFieldByName("city"), nullptr); +} + +// Verifies that adaptive inference returns a complete physical row schema to the write-plan +// factory, and that committed evidence is reused only after the preceding file completes. +TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceReturnsCompletePhysicalSchema) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + std::vector> samples = { + BuildBatch({R"({"age": 35, "city": "Chicago"})"})}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr expected_variant_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("age", arrow::int64()), arrow::field("city", arrow::utf8())}))); + auto expected_schema = + arrow::schema({schema_->field(0), schema_->field(1)->WithType(expected_variant_type)}, + schema_->metadata()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(first, nullptr); + ASSERT_TRUE(first->GetPhysicalSchema()->Equals(*expected_schema)) + << first->GetPhysicalSchema()->ToString(); + + ASSERT_OK(factory->OnFileCompleted(first)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + factory->CreateConverter("parquet", {})); + ASSERT_NE(second, nullptr); + ASSERT_TRUE(second->GetPhysicalSchema()->Equals(*expected_schema)) + << second->GetPhysicalSchema()->ToString(); + ASSERT_OK(factory->OnFileCompleted(second)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceUsesAdmissionAndRetentionThresholds) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.maxInferBufferRow", "10"}, + {"variant.shredding.adaptive.maxInferBufferRow", "10"}, + {"variant.shredding.minFieldCardinalityRatio", "0.4"}, + {"variant.shredding.adaptive.retentionRatio", "0.2"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + auto typed_value = [](const std::shared_ptr& converter) { + const auto& variant_type = static_cast( + *converter->GetPhysicalSchema()->GetFieldByName("v")->type()); + return variant_type.GetFieldByName("typed_value"); + }; + + std::vector> first_samples = { + BuildBatch({R"({"legacy":"v","stable":1})", R"({"legacy":"v","stable":1})", + R"({"legacy":"v","stable":1})", R"({"legacy":"v","stable":1})", + R"({"legacy":"v","stable":1})", R"({"stable":1})", R"({"stable":1})", + R"({"stable":1})", R"({"stable":1})", R"({"stable":1})"})}; + ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples)); + auto first_typed = typed_value(first); + ASSERT_NE(nullptr, first_typed); + const auto& first_struct = static_cast(*first_typed->type()); + ASSERT_TRUE(ShreddedTypedValue(first_struct, "legacy")->Equals(*arrow::utf8())); + ASSERT_TRUE(ShreddedTypedValue(first_struct, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(first)); + + std::vector> second_samples = { + BuildBatch({R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})", + R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})", + R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})", + R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})", + R"({"emerging":true,"stable":2})", R"({"stable":2})"})}; + ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", second_samples)); + auto second_typed = typed_value(second); + ASSERT_NE(nullptr, second_typed); + const auto& second_struct = static_cast(*second_typed->type()); + ASSERT_TRUE(ShreddedTypedValue(second_struct, "emerging")->Equals(*arrow::boolean())); + ASSERT_TRUE(ShreddedTypedValue(second_struct, "legacy")->Equals(*arrow::utf8())); + ASSERT_TRUE(ShreddedTypedValue(second_struct, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(second)); + + std::vector> third_samples = { + BuildBatch({R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", + R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", + R"({"stable":3})", R"({"stable":3})"})}; + ASSERT_OK_AND_ASSIGN(auto third, factory->CreateConverter("parquet", third_samples)); + auto third_typed = typed_value(third); + ASSERT_NE(nullptr, third_typed); + const auto& third_struct = static_cast(*third_typed->type()); + ASSERT_TRUE(ShreddedTypedValue(third_struct, "emerging")->Equals(*arrow::boolean())); + ASSERT_EQ(nullptr, third_struct.GetFieldByName("legacy")); + ASSERT_TRUE(ShreddedTypedValue(third_struct, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(third)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceOnShortRolledFile) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.maxInferBufferRow", "4"}, + {"variant.shredding.adaptive.maxInferBufferRow", "4"}, + {"variant.shredding.minFieldCardinalityRatio", "0.4"}, + {"variant.shredding.adaptive.retentionRatio", "0.2"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + + std::vector> first_samples = { + BuildBatch({R"({"legacy":"a","stable":1})", R"({"legacy":"b","stable":2})", + R"({"legacy":"c","stable":3})", R"({"legacy":"d","stable":4})"})}; + ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples)); + ASSERT_OK(factory->OnFileCompleted(first)); + + std::vector> short_samples = { + BuildBatch({R"({"emerging":true,"stable":5})", R"({"emerging":false,"stable":6})", + R"({"emerging":true,"stable":7})"})}; + ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", short_samples)); + const auto& variant_type = static_cast( + *second->GetPhysicalSchema()->GetFieldByName("v")->type()); + const auto& typed_object = + static_cast(*variant_type.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(typed_object, "emerging")->Equals(*arrow::boolean())); + ASSERT_TRUE(ShreddedTypedValue(typed_object, "legacy")->Equals(*arrow::utf8())); + ASSERT_TRUE(ShreddedTypedValue(typed_object, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(second)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, + AdaptiveInferenceWidensScalarSelectedFromPriorEvidence) { + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("first")), + DataField(3, VariantTypeUtils::ToArrowField("second"))}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.maxSchemaWidth", "6"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + auto build_batch = [&](const std::vector& first_jsons, + const std::vector& second_jsons) { + auto first = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1), + first_jsons, pool_); + EXPECT_TRUE(first.ok()) << first.status().ToString(); + auto second = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2), + second_jsons, pool_); + EXPECT_TRUE(second.ok()) << second.status().ToString(); + return std::shared_ptr( + arrow::StructArray::Make( + {first.value()->field(0), first.value()->field(1), second.value()->field(1)}, + schema->fields()) + .ValueOrDie()); + }; + + std::vector> first_samples = { + build_batch({R"({"a":1,"b":2})"}, {R"({"historical":12345})"})}; + ASSERT_OK_AND_ASSIGN(auto initial, factory->CreateConverter("parquet", first_samples)); + const auto& initial_first = static_cast( + *initial->GetPhysicalSchema()->GetFieldByName("first")->type()); + const auto& initial_first_typed = + static_cast(*initial_first.GetFieldByName("typed_value")->type()); + ASSERT_NE(nullptr, initial_first_typed.GetFieldByName("a")); + ASSERT_NE(nullptr, initial_first_typed.GetFieldByName("b")); + const auto& initial_second = static_cast( + *initial->GetPhysicalSchema()->GetFieldByName("second")->type()); + ASSERT_EQ(nullptr, initial_second.GetFieldByName("typed_value")); + ASSERT_OK(factory->OnFileCompleted(initial)); + + std::vector> adaptive_samples = {build_batch({"1"}, {nullptr})}; + ASSERT_OK_AND_ASSIGN(auto adaptive, factory->CreateConverter("parquet", adaptive_samples)); + const auto& adaptive_first = static_cast( + *adaptive->GetPhysicalSchema()->GetFieldByName("first")->type()); + ASSERT_TRUE(adaptive_first.GetFieldByName("typed_value")->type()->Equals(*arrow::int64())); + const auto& adaptive_second = static_cast( + *adaptive->GetPhysicalSchema()->GetFieldByName("second")->type()); + const auto& adaptive_second_typed = static_cast( + *adaptive_second.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(adaptive_second_typed, "historical")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(adaptive)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceWithNestedVariant) { + auto nested_variant = VariantTypeUtils::ToArrowField("payload"); + auto nested_field = arrow::field( + "nested", arrow::struct_({arrow::field("label", arrow::utf8()), nested_variant})); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, nested_field)}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.maxInferBufferRow", "2"}, + {"variant.shredding.adaptive.maxInferBufferRow", "2"}, + {"variant.shredding.minFieldCardinalityRatio", "0.4"}, + {"variant.shredding.adaptive.retentionRatio", "0.2"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + auto build_nested_batch = [&](const std::vector& labels, + const std::vector& jsons) { + auto variants = + VariantTestData::BuildVariantBatch(schema->field(0), nested_variant, jsons, pool_); + EXPECT_TRUE(variants.ok()) << variants.status().ToString(); + arrow::StringBuilder label_builder; + for (const char* label : labels) { + EXPECT_TRUE(label_builder.Append(label).ok()); + } + std::shared_ptr label_array; + EXPECT_TRUE(label_builder.Finish(&label_array).ok()); + auto nested = arrow::StructArray::Make({label_array, variants.value()->field(1)}, + nested_field->type()->fields()) + .ValueOrDie(); + return std::shared_ptr( + arrow::StructArray::Make({variants.value()->field(0), nested}, schema->fields()) + .ValueOrDie()); + }; + auto nested_typed_object = [](const std::shared_ptr& converter) { + const auto& physical_nested = static_cast( + *converter->GetPhysicalSchema()->GetFieldByName("nested")->type()); + const auto& physical_variant = static_cast( + *physical_nested.GetFieldByName("payload")->type()); + return std::static_pointer_cast( + physical_variant.GetFieldByName("typed_value")->type()); + }; + + std::vector> first_samples = {build_nested_batch( + {"first", "second"}, {R"({"legacy":"a","stable":1})", R"({"legacy":"b","stable":2})"})}; + ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples)); + auto first_typed = nested_typed_object(first); + ASSERT_TRUE(ShreddedTypedValue(*first_typed, "legacy")->Equals(*arrow::utf8())); + ASSERT_TRUE(ShreddedTypedValue(*first_typed, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(first)); + + std::vector> second_samples = {build_nested_batch( + {"third", "fourth"}, + {R"({"emerging":true,"stable":3})", R"({"emerging":false,"stable":4})"})}; + ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", second_samples)); + auto second_typed = nested_typed_object(second); + ASSERT_TRUE(ShreddedTypedValue(*second_typed, "emerging")->Equals(*arrow::boolean())); + ASSERT_TRUE(ShreddedTypedValue(*second_typed, "legacy")->Equals(*arrow::utf8())); + ASSERT_TRUE(ShreddedTypedValue(*second_typed, "stable")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(second)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchemaWithoutSamples) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + // With no useful samples the complete physical plan keeps the Variant untyped. + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_NE(converter, nullptr); + const auto& physical_type = static_cast( + *converter->GetPhysicalSchema()->GetFieldByName("v")->type()); + ASSERT_EQ(physical_type.GetFieldByName("typed_value"), nullptr); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, MultipleVariantFieldsInferredIndependently) { + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v1")), + DataField(3, VariantTypeUtils::ToArrowField("v2"))}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.maxInferBufferRow", "2"}, + {"variant.shredding.adaptive.maxInferBufferRow", "2"}, + {"variant.shredding.minFieldCardinalityRatio", "0.4"}, + {"variant.shredding.adaptive.retentionRatio", "0.2"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + + auto v1 = VariantTestData::BuildVariantBatch( + schema->field(0), schema->field(1), {R"({"name":"Alice"})", R"({"name":"Bob"})"}, pool_); + ASSERT_TRUE(v1.ok()) << v1.status().ToString(); + auto v2 = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2), + {R"({"age":30})", R"({"age":25})"}, pool_); + ASSERT_TRUE(v2.ok()) << v2.status().ToString(); + std::shared_ptr batch = + arrow::StructArray::Make({v1.value()->field(0), v1.value()->field(1), v2.value()->field(1)}, + schema->fields()) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto converter, factory->CreateConverter("parquet", {batch})); + + const auto& v1_physical = static_cast( + *converter->GetPhysicalSchema()->GetFieldByName("v1")->type()); + const auto& v1_typed = + static_cast(*v1_physical.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(v1_typed, "name")->Equals(*arrow::utf8())); + + const auto& v2_physical = static_cast( + *converter->GetPhysicalSchema()->GetFieldByName("v2")->type()); + const auto& v2_typed = + static_cast(*v2_physical.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(v2_typed, "age")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(converter)); + + auto next_v1 = VariantTestData::BuildVariantBatch( + schema->field(0), schema->field(1), + {R"({"emerging":true,"name":"Carol"})", R"({"emerging":false,"name":"Dave"})"}, pool_); + ASSERT_TRUE(next_v1.ok()) << next_v1.status().ToString(); + auto next_v2 = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2), + {R"({"age":40})", R"({"age":45})"}, pool_); + ASSERT_TRUE(next_v2.ok()) << next_v2.status().ToString(); + std::shared_ptr next_batch = + arrow::StructArray::Make( + {next_v1.value()->field(0), next_v1.value()->field(1), next_v2.value()->field(1)}, + schema->fields()) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto adaptive, factory->CreateConverter("parquet", {next_batch})); + const auto& adaptive_v1 = static_cast( + *adaptive->GetPhysicalSchema()->GetFieldByName("v1")->type()); + const auto& adaptive_v1_typed = + static_cast(*adaptive_v1.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(adaptive_v1_typed, "emerging")->Equals(*arrow::boolean())); + ASSERT_TRUE(ShreddedTypedValue(adaptive_v1_typed, "name")->Equals(*arrow::utf8())); + const auto& adaptive_v2 = static_cast( + *adaptive->GetPhysicalSchema()->GetFieldByName("v2")->type()); + const auto& adaptive_v2_typed = + static_cast(*adaptive_v2.GetFieldByName("typed_value")->type()); + ASSERT_TRUE(ShreddedTypedValue(adaptive_v2_typed, "age")->Equals(*arrow::int64())); + ASSERT_OK(factory->OnFileCompleted(adaptive)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, SharedWidthBudgetAcrossColumns) { + // Two variant columns share one maxSchemaWidth budget (as in Java): with a small limit the + // first column consumes it and the second column stays unshredded. + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v1")), + DataField(3, VariantTypeUtils::ToArrowField("v2"))}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.maxSchemaWidth", "3"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + + auto build_batch = [&](const std::vector& v1_jsons, + const std::vector& v2_jsons) { + auto v1 = + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1), v1_jsons, pool_); + EXPECT_TRUE(v1.ok()) << v1.status().ToString(); + auto v2 = + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2), v2_jsons, pool_); + EXPECT_TRUE(v2.ok()) << v2.status().ToString(); + auto batch = arrow::StructArray::Make( + {v1.value()->field(0), v1.value()->field(1), v2.value()->field(1)}, + {schema->field(0), schema->field(1), schema->field(2)}) + .ValueOrDie(); + return std::shared_ptr(batch); + }; + std::vector> samples = { + build_batch({R"({"a": 1, "b": 2})"}, {R"({"c": 3})"})}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + const auto& physical = converter->GetPhysicalSchema(); + // v1 got the budget and is shredded; v2 exceeded it and stays unshredded. + const auto& v1_type = + static_cast(*physical->GetFieldByName("v1")->type()); + ASSERT_NE(v1_type.GetFieldByName("typed_value"), nullptr); + const auto& v2_type = + static_cast(*physical->GetFieldByName("v2")->type()); + ASSERT_EQ(v2_type.GetFieldByName("typed_value"), nullptr); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, NestedVariantInsideStruct) { + // A VARIANT nested inside a ROW column is discovered, inferred and shredded (as in Java); + // top-level detection also recurses through structs. + auto nested_variant = VariantTypeUtils::ToArrowField("nv"); + auto struct_field = arrow::field("s", arrow::struct_({nested_variant})); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, struct_field)}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant_batch, + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1)->type()->field(0), + {R"({"age": 35, "city": "Chicago"})"}, pool_)); + std::shared_ptr struct_column = + arrow::StructArray::Make({variant_batch->field(1)}, {schema->field(1)->type()->field(0)}) + .ValueOrDie(); + std::shared_ptr batch = + arrow::StructArray::Make({variant_batch->field(0), struct_column}, + {schema->field(0), schema->field(1)}) + .ValueOrDie(); + std::vector> samples = {batch}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + const auto& physical = converter->GetPhysicalSchema(); + const auto& physical_struct = + static_cast(*physical->GetFieldByName("s")->type()); + const auto& physical_variant = + static_cast(*physical_struct.GetFieldByName("nv")->type()); + auto typed_value = physical_variant.GetFieldByName("typed_value"); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + ASSERT_NE(typed_struct.GetFieldByName("city"), nullptr); + + // Child slot contents under a null struct row are unspecified in Arrow: a row whose parent + // struct is null must be skipped by sampling and shredded to null by conversion, without + // decoding the (here invalid) variant bytes underneath. + arrow::BinaryBuilder garbage_builder; + ASSERT_TRUE(garbage_builder.Append("not a variant").ok()); + std::shared_ptr garbage; + ASSERT_TRUE(garbage_builder.Finish(&garbage).ok()); + std::shared_ptr garbage_variant = + arrow::StructArray::Make({garbage, garbage}, {nested_variant->type()->field(0), + nested_variant->type()->field(1)}) + .ValueOrDie(); + // Mark the single parent row null (a zeroed bitmap) while the child slots keep their bytes. + auto null_struct_column = std::make_shared( + struct_field->type(), 1, arrow::ArrayVector{garbage_variant}, + arrow::AllocateEmptyBitmap(1).ValueOrDie(), 1); + std::shared_ptr garbage_batch = + arrow::StructArray::Make({variant_batch->field(0), null_struct_column}, + {schema->field(0), schema->field(1)}) + .ValueOrDie(); + + // Sampling sees no usable value: the complete physical plan keeps the Variant untyped. + std::vector> garbage_samples = {garbage_batch}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr unshredded_converter, + factory->CreateConverter("parquet", garbage_samples)); + ASSERT_NE(unshredded_converter, nullptr); + const auto& unshredded_struct = static_cast( + *unshredded_converter->GetPhysicalSchema()->GetFieldByName("s")->type()); + const auto& unshredded_variant = + static_cast(*unshredded_struct.GetFieldByName("nv")->type()); + ASSERT_EQ(unshredded_variant.GetFieldByName("typed_value"), nullptr); + + // Conversion with the previously inferred plan shreds the row to null instead of failing. + auto c_garbage_batch = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*garbage_batch, c_garbage_batch.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_garbage_batch.get())); + auto physical_struct_type = arrow::struct_(physical->fields()); + std::shared_ptr physical_array = + arrow::ImportArray(c_physical.get(), physical_struct_type).ValueOrDie(); + const auto& physical_row = static_cast(*physical_array); + const auto& converted_struct = static_cast(*physical_row.field(1)); + ASSERT_TRUE(converted_struct.IsNull(0)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, ConfiguredSchemaMatchingNoColumn) { + // A configured schema naming no variant column writes the file unshredded (as in Java) + // instead of failing. + const char* shredding_schema_json = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "not_a_column", + "type": {"type": "ROW", "fields": [{"id": 1, "name": "a", "type": "INT"}]} + } ] + })"; + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.shreddingSchema", shredding_schema_json}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_NE(converter, nullptr); + ASSERT_TRUE(converter->GetPhysicalSchema()->Equals(*schema_)); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_shredding_writer.cpp b/src/paimon/common/data/variant/variant_shredding_writer.cpp new file mode 100644 index 00000000..bb7c412e --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_writer.cpp @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_shredding_writer.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +// Rescales `unscaled` by `10^power`, failing when the result would exceed the 38-digit +// limit. The precision check must precede each multiplication: a 38-digit value times 10 +// overflows the signed 128-bit representation before a post-check could reject it. +Result<__int128_t> ScaleUpUnscaled(__int128_t unscaled, int32_t power) { + __int128_t result = unscaled; + for (int32_t i = 0; i < power; ++i) { + VariantDecimal probe{result, 0}; + if (probe.Precision() >= VariantDefs::kMaxDecimal16Precision) { + return Status::Invalid("decimal overflow while rescaling"); + } + result *= 10; + } + return result; +} + +Status AppendDecimalTo(__int128_t unscaled, arrow::ArrayBuilder* builder) { + auto* decimal_builder = static_cast(builder); + arrow::Decimal128 value(static_cast(unscaled >> 64), + static_cast(static_cast<__uint128_t>(unscaled))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(decimal_builder->Append(value)); + return Status::OK(); +} + +} // namespace + +VariantShreddedColumnWriter::VariantShreddedColumnWriter( + const std::shared_ptr& schema, + std::unique_ptr&& root_builder) + : schema_(schema), root_builder_(std::move(root_builder)) {} + +Result> VariantShreddedColumnWriter::Create( + const std::shared_ptr& schema, + const std::shared_ptr& physical_type, arrow::MemoryPool* pool) { + if (!schema || schema->top_level_metadata_idx < 0) { + return Status::Invalid("variant shredding schema must contain a top-level metadata field"); + } + std::unique_ptr root_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(pool, physical_type, &root_builder)); + if (root_builder->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid("variant shredded physical type must be a struct"); + } + auto writer = std::unique_ptr( + new VariantShreddedColumnWriter(schema, std::move(root_builder))); + PAIMON_RETURN_NOT_OK(BuildNode( + schema, static_cast(writer->root_builder_.get()), &writer->root_)); + return writer; +} + +Status VariantShreddedColumnWriter::BuildNode(const std::shared_ptr& schema, + arrow::StructBuilder* group, Node* node) { + node->schema = schema.get(); + node->group = group; + if (group->num_children() != schema->num_fields) { + return Status::Invalid( + fmt::format("variant shredded builder has {} children but the schema has {} fields", + group->num_children(), schema->num_fields)); + } + if (schema->top_level_metadata_idx >= 0) { + node->metadata = static_cast( + group->field_builder(schema->top_level_metadata_idx)); + } + if (schema->variant_idx >= 0) { + node->value = static_cast(group->field_builder(schema->variant_idx)); + } + if (schema->typed_idx >= 0) { + arrow::ArrayBuilder* typed_builder = group->field_builder(schema->typed_idx); + if (schema->has_object_schema) { + node->typed_object = static_cast(typed_builder); + node->object_children.resize(schema->object_schema.size()); + for (size_t i = 0; i < schema->object_schema.size(); ++i) { + auto* child_group = static_cast( + node->typed_object->field_builder(static_cast(i))); + PAIMON_RETURN_NOT_OK(BuildNode(schema->object_schema[i].schema, child_group, + &node->object_children[i])); + } + } else if (schema->array_schema) { + node->typed_list = static_cast(typed_builder); + node->array_element = std::make_unique(); + auto* element_group = + static_cast(node->typed_list->value_builder()); + PAIMON_RETURN_NOT_OK( + BuildNode(schema->array_schema, element_group, node->array_element.get())); + } else if (schema->scalar_schema) { + node->typed_scalar = typed_builder; + } else { + return Status::Invalid("variant shredding schema typed_value has no schema"); + } + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::Append(const GenericVariant& variant) { + return AppendVariantNode(variant, &root_); +} + +Status VariantShreddedColumnWriter::AppendNull() { + PAIMON_RETURN_NOT_OK_FROM_ARROW(root_.group->AppendNull()); + return Status::OK(); +} + +Result> VariantShreddedColumnWriter::Finish() { + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(root_builder_->Finish(&array)); + return array; +} + +Status VariantShreddedColumnWriter::AppendVariantNode(const GenericVariant& variant, Node* node) { + const VariantSchema& schema = *node->schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->group->Append()); + if (schema.top_level_metadata_idx >= 0) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->metadata->Append(variant.Metadata())); + } + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant.GetType()); + if (schema.array_schema != nullptr && variant_type == VariantValueType::kArray) { + // The array element is always a struct containing untyped and typed fields. + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->Append()); + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ArraySize()); + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant.GetElementAtIndex(i)); + PAIMON_RETURN_NOT_OK(AppendVariantNode(*element, node->array_element.get())); + } + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else if (schema.has_object_schema && variant_type == VariantValueType::kObject) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->Append()); + // Collect any field that exists in the variant, but not in the shredding schema, into a + // residual variant object that shares the top-level metadata. + VariantBuilder residual(/*allow_duplicate_keys=*/false); + std::vector field_entries; + std::vector matched(node->object_children.size(), false); + int32_t start = residual.GetWritePos(); + PAIMON_ASSIGN_OR_RAISE(int32_t object_size, variant.ObjectSize()); + for (int32_t i = 0; i < object_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant.GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant("an object field is missing"); + } + auto it = schema.object_schema_map.find(field->key); + if (it != schema.object_schema_map.end()) { + PAIMON_RETURN_NOT_OK( + AppendVariantNode(*field->value, &node->object_children[it->second])); + matched[it->second] = true; + } else { + // The field is not shredded. Put it in the untyped value column. The shallow + // append is needed for correctness, since the metadata ids must stay unchanged. + PAIMON_ASSIGN_OR_RAISE(int32_t id, variant.GetDictionaryIdAtIndex(i)); + field_entries.emplace_back(field->key, id, residual.GetWritePos() - start); + PAIMON_RETURN_NOT_OK( + residual.ShallowAppendVariant(field->value->RawValue(), field->value->Pos())); + } + } + // Set missing fields to non-null with all fields set to null. + for (size_t i = 0; i < matched.size(); ++i) { + if (!matched[i]) { + PAIMON_RETURN_NOT_OK(AppendMissingNode(&node->object_children[i])); + } + } + if (residual.GetWritePos() != start) { + PAIMON_RETURN_NOT_OK(residual.FinishWritingObject(start, &field_entries)); + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for residual fields"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(residual.ValueWithoutMetadata())); + } else if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else if (schema.scalar_schema.has_value()) { + bool shredded = false; + PAIMON_RETURN_NOT_OK(TryTypedShred(variant, variant_type, node, &shredded)); + if (shredded) { + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for untyped values"); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(value)); + } + } else { + if (node->typed_list != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->AppendNull()); + } else if (node->typed_object != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->AppendNull()); + } else if (node->typed_scalar != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + } + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for untyped values"); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(value)); + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::AppendMissingNode(Node* node) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->group->Append()); + if (node->metadata != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->metadata->AppendNull()); + } + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + if (node->typed_list != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->AppendNull()); + } else if (node->typed_object != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->AppendNull()); + } else if (node->typed_scalar != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::TryTypedShred(const GenericVariant& variant, + VariantValueType variant_type, Node* node, + bool* shredded) { + const VariantSchema::ScalarType& target = node->schema->scalar_schema.value(); + *shredded = false; + switch (variant_type) { + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + switch (target.kind) { + // Check that the target type can hold the actual value. + case VariantSchema::ScalarKind::kByte: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kShort: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kInt: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kLong: + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + break; + case VariantSchema::ScalarKind::kDecimal: { + // If the integer can fit in the given decimal precision, allow it. + auto scaled = ScaleUpUnscaled(value, target.scale); + if (scaled.ok()) { + VariantDecimal probe{scaled.value(), target.scale}; + if (probe.Precision() <= target.precision) { + PAIMON_RETURN_NOT_OK( + AppendDecimalTo(scaled.value(), node->typed_scalar)); + *shredded = true; + } + } + break; + } + default: + break; + } + break; + } + case VariantValueType::kDecimal: { + if (target.kind == VariantSchema::ScalarKind::kDecimal) { + // Use the original scale so that scale information is retained. + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, + VariantBinaryUtil::GetDecimalWithOriginalScale( + variant.RawValue(), variant.Pos())); + if (value.Precision() <= target.precision && value.scale == target.scale) { + PAIMON_RETURN_NOT_OK(AppendDecimalTo(value.unscaled, node->typed_scalar)); + *shredded = true; + break; + } + // Convert to the target scale, and see if it fits without losing information. + int32_t scale_diff = target.scale - value.scale; + __int128_t rescaled = value.unscaled; + bool exact = true; + if (scale_diff >= 0) { + auto scaled = ScaleUpUnscaled(value.unscaled, scale_diff); + if (scaled.ok()) { + rescaled = scaled.value(); + } else { + exact = false; + } + } else { + for (int32_t i = 0; i < -scale_diff && exact; ++i) { + if (rescaled % 10 != 0) { + exact = false; + } else { + rescaled /= 10; + } + } + } + if (exact) { + VariantDecimal probe{rescaled, target.scale}; + if (probe.Precision() <= target.precision) { + PAIMON_RETURN_NOT_OK(AppendDecimalTo(rescaled, node->typed_scalar)); + *shredded = true; + } + } + } else if (target.kind == VariantSchema::ScalarKind::kByte || + target.kind == VariantSchema::ScalarKind::kShort || + target.kind == VariantSchema::ScalarKind::kInt || + target.kind == VariantSchema::ScalarKind::kLong) { + // Check if the decimal happens to be an integer. + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, variant.GetDecimal()); + if (value.scale > 0) { + break; + } + auto scaled = ScaleUpUnscaled(value.unscaled, -value.scale); + if (!scaled.ok()) { + break; + } + __int128_t integral = scaled.value(); + if (integral != static_cast(integral)) { + break; + } + auto long_value = static_cast(integral); + switch (target.kind) { + case VariantSchema::ScalarKind::kByte: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kShort: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kInt: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + default: + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(long_value)); + *shredded = true; + break; + } + } + break; + } + case VariantValueType::kBoolean: { + if (target.kind == VariantSchema::ScalarKind::kBoolean) { + PAIMON_ASSIGN_OR_RAISE(bool value, variant.GetBoolean()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kString: { + if (target.kind == VariantSchema::ScalarKind::kString) { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.GetString()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kDouble: { + if (target.kind == VariantSchema::ScalarKind::kDouble) { + PAIMON_ASSIGN_OR_RAISE(double value, variant.GetDouble()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kFloat: { + if (target.kind == VariantSchema::ScalarKind::kFloat) { + PAIMON_ASSIGN_OR_RAISE(float value, variant.GetFloat()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kDate: { + if (target.kind == VariantSchema::ScalarKind::kDate) { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + } + case VariantValueType::kBinary: { + if (target.kind == VariantSchema::ScalarKind::kBinary) { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.GetBinary()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + default: + // TIMESTAMP/TIMESTAMP_NTZ/UUID typed columns are not producible by the configured + // shredding schema; the value stays in the untyped column. + break; + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_writer.h b/src/paimon/common/data/variant/variant_shredding_writer.h new file mode 100644 index 00000000..81723c28 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_writer.h @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class ArrayBuilder; +class BinaryBuilder; +class DataType; +class ListBuilder; +class MemoryPool; +class StructBuilder; +} // namespace arrow + +namespace paimon { + +/// Shreds variant values of one column into a physical shredded Arrow array, implementing the +/// `castShredded` algorithm of the parquet-format VariantShredding.md specification (mirroring +/// the Java `VariantShreddingWriter`). Decimals and integers are allowed to shred to numerically +/// equivalent values of a different scale (`allowNumericScaleChanges` in Java is always true). +class VariantShreddedColumnWriter { + public: + /// Creates a writer for one variant column. + /// + /// @param schema The shredding schema of the column. + /// @param physical_type The physical shredded struct type + /// (`VariantShreddingUtils::VariantShreddingSchema` output). + /// @param pool The Arrow memory pool used by the builders. + static Result> Create( + const std::shared_ptr& schema, + const std::shared_ptr& physical_type, arrow::MemoryPool* pool); + + /// Shreds one variant value and appends the result row. + Status Append(const GenericVariant& variant); + + /// Appends a null variant row. + Status AppendNull(); + + /// Finishes and returns the shredded array of all appended rows. + Result> Finish(); + + private: + /// Builder handles of one shredding schema node (a `metadata`/`value`/`typed_value` group). + struct Node { + const VariantSchema* schema = nullptr; + arrow::StructBuilder* group = nullptr; + arrow::BinaryBuilder* metadata = nullptr; + arrow::BinaryBuilder* value = nullptr; + // Exactly one of the following is set when `schema->typed_idx >= 0`. + arrow::ArrayBuilder* typed_scalar = nullptr; + arrow::ListBuilder* typed_list = nullptr; + arrow::StructBuilder* typed_object = nullptr; + std::vector object_children; + std::unique_ptr array_element; + }; + + VariantShreddedColumnWriter(const std::shared_ptr& schema, + std::unique_ptr&& root_builder); + + static Status BuildNode(const std::shared_ptr& schema, + arrow::StructBuilder* group, Node* node); + + Status AppendVariantNode(const GenericVariant& variant, Node* node); + + /// Appends a missing object field: the group is present with all its children set to null. + Status AppendMissingNode(Node* node); + + /// Tries to append the variant as a typed scalar. Sets `*shredded` to whether it succeeded + /// (on failure nothing is appended). + Status TryTypedShred(const GenericVariant& variant, VariantValueType variant_type, Node* node, + bool* shredded); + + std::shared_ptr schema_; + std::unique_ptr root_builder_; + Node root_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_test.cpp b/src/paimon/common/data/variant/variant_test.cpp new file mode 100644 index 00000000..f35f5d8e --- /dev/null +++ b/src/paimon/common/data/variant/variant_test.cpp @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/variant.h" + +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantPublicApiTest : public ::testing::Test { + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(VariantPublicApiTest, FromJsonAndAccessors) { + ASSERT_OK_AND_ASSIGN(auto variant, + Variant::FromJson("{\"age\": 35, \"city\": \"Hangzhou\"}", pool_)); + ASSERT_GT(variant->Value().size(), 0); + ASSERT_GT(variant->Metadata().size(), 0); + ASSERT_EQ(variant->SizeInBytes(), + static_cast(variant->Value().size() + variant->Metadata().size())); + ASSERT_OK_AND_ASSIGN(std::string json, variant->ToJson()); + ASSERT_EQ(json, "{\"age\":35,\"city\":\"Hangzhou\"}"); + + ASSERT_OK_AND_ASSIGN( + auto rebuilt, + Variant::Create(variant->Value().data(), variant->Value().size(), + variant->Metadata().data(), variant->Metadata().size(), pool_)); + ASSERT_OK_AND_ASSIGN(std::string rebuilt_json, rebuilt->ToJson()); + ASSERT_EQ(rebuilt_json, json); +} + +TEST_F(VariantPublicApiTest, VariantGet) { + ASSERT_OK_AND_ASSIGN(auto variant, + Variant::FromJson("{\"age\": 35, \"city\": \"Hangzhou\"}", pool_)); + VariantCastArgs cast_args; + cast_args.fail_on_error = false; + { + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", arrow::int64()), target.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::optional literal, + variant->VariantGet("$.age", target.get(), cast_args)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue(), 35); + } + { + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", arrow::utf8()), target.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::optional literal, + variant->VariantGet("$.missing", target.get(), cast_args)); + ASSERT_FALSE(literal.has_value()); + } + ASSERT_OK_AND_ASSIGN(std::optional sub_json, variant->VariantGetJson("$")); + ASSERT_TRUE(sub_json.has_value()); + ASSERT_EQ(*sub_json, "{\"age\":35,\"city\":\"Hangzhou\"}"); + ASSERT_OK_AND_ASSIGN(std::optional missing_json, + variant->VariantGetJson("$.missing")); + ASSERT_FALSE(missing_json.has_value()); +} + +TEST_F(VariantPublicApiTest, ArrowField) { + ASSERT_OK_AND_ASSIGN(auto c_field, Variant::ArrowField("v", /*nullable=*/true)); + auto imported = arrow::ImportField(c_field.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + std::shared_ptr field = imported.ValueOrDie(); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_TRUE(field->nullable()); + ASSERT_TRUE(field->type()->Equals(VariantTypeUtils::UnshreddedStructType())); +} + +TEST_F(VariantPublicApiTest, VariantGetArrow) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr variant, + Variant::FromJson("{\"user\": {\"name\": \"Paimon\"}}", pool_)); + auto target_type = arrow::struct_({arrow::field("name", arrow::utf8())}); + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", target_type), target.get()).ok()); + VariantCastArgs cast_args; + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_array, + variant->VariantGetArrow("$.user", target.get(), cast_args)); + auto imported = arrow::ImportArray(c_array.get(), target_type); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + std::shared_ptr array = imported.ValueOrDie(); + ASSERT_EQ(array->length(), 1); + const auto& row = static_cast(*array); + ASSERT_EQ(static_cast(*row.field(0)).GetString(0), "Paimon"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_type_utils.cpp b/src/paimon/common/data/variant/variant_type_utils.cpp new file mode 100644 index 00000000..6bb494e1 --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils.cpp @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_type_utils.h" + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/types/data_field.h" + +namespace paimon { + +bool VariantTypeUtils::IsVariantField(const std::shared_ptr& field) { + if (field->type()->id() != arrow::Type::STRUCT) { + return false; + } + if (!field->HasMetadata()) { + return false; + } + return IsVariantMetadata(field->metadata()); +} + +bool VariantTypeUtils::IsUnshreddedVariantType(const std::shared_ptr& type) { + if (type == nullptr || type->id() != arrow::Type::STRUCT || type->num_fields() != 2) { + return false; + } + const auto& value_field = type->field(0); + const auto& metadata_field = type->field(1); + return value_field->name() == VariantDefs::kValueFieldName && + value_field->type()->id() == arrow::Type::BINARY && !value_field->nullable() && + metadata_field->name() == VariantDefs::kMetadataFieldName && + metadata_field->type()->id() == arrow::Type::BINARY && !metadata_field->nullable(); +} + +bool VariantTypeUtils::IsVariantMetadata( + const std::shared_ptr& metadata) { + if (!metadata) { + return false; + } + auto extension_name = metadata->Get(VariantDefs::kExtensionTypeKey); + return extension_name.ok() && *extension_name == VariantDefs::kExtensionTypeValue; +} + +std::shared_ptr VariantTypeUtils::UnshreddedStructType() { + auto value_field = + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/false, + arrow::KeyValueMetadata::Make({DataField::FIELD_ID}, + {std::to_string(VariantDefs::kValueFieldId)})); + auto metadata_field = + arrow::field(VariantDefs::kMetadataFieldName, arrow::binary(), /*nullable=*/false, + arrow::KeyValueMetadata::Make( + {DataField::FIELD_ID}, {std::to_string(VariantDefs::kMetadataFieldId)})); + return arrow::struct_({value_field, metadata_field}); +} + +std::shared_ptr VariantTypeUtils::ToArrowField( + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + metadata[VariantDefs::kExtensionTypeKey] = VariantDefs::kExtensionTypeValue; + return arrow::field(field_name, UnshreddedStructType(), nullable, + std::make_shared(metadata)); +} + +Status VariantTypeUtils::ValidateVariantShape(const std::shared_ptr& field) { + const auto& type = field->type(); + if (type->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("Variant field '{}' must be a struct, but got {}", + field->name(), type->ToString())); + } + const auto& struct_type = std::static_pointer_cast(type); + if (struct_type->num_fields() != 2) { + return Status::Invalid( + fmt::format("Variant field '{}' must be a struct, " + "but got {}", + field->name(), type->ToString())); + } + const auto& value_field = struct_type->field(0); + const auto& metadata_field = struct_type->field(1); + if (value_field->name() != VariantDefs::kValueFieldName || + value_field->type()->id() != arrow::Type::BINARY || value_field->nullable() || + metadata_field->name() != VariantDefs::kMetadataFieldName || + metadata_field->type()->id() != arrow::Type::BINARY || metadata_field->nullable()) { + return Status::Invalid( + fmt::format("Variant field '{}' must be a struct, but got {}", + field->name(), type->ToString())); + } + return Status::OK(); +} + +bool VariantTypeUtils::ContainsVariantField(const std::shared_ptr& field) { + if (IsVariantField(field)) { + return true; + } + for (const auto& child : field->type()->fields()) { + if (ContainsVariantField(child)) { + return true; + } + } + return false; +} + +bool VariantTypeUtils::ContainsVariantField(const std::shared_ptr& schema) { + for (const auto& field : schema->fields()) { + if (ContainsVariantField(field)) { + return true; + } + } + return false; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_type_utils.h b/src/paimon/common/data/variant/variant_type_utils.h new file mode 100644 index 00000000..89b8201e --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace arrow { +class DataType; +class Field; +class KeyValueMetadata; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Utils for the Paimon Variant type, whose underlying Arrow representation is +/// `struct` marked with the +/// `paimon.extension.type = paimon.type.variant` field metadata (see `VariantDefs`). +class PAIMON_EXPORT VariantTypeUtils { + public: + VariantTypeUtils() = delete; + ~VariantTypeUtils() = delete; + + /// Whether the field is a Paimon Variant field (a struct with the variant metadata marker). + static bool IsVariantField(const std::shared_ptr& field); + + /// Whether `type` is the unshredded variant physical type + /// `struct` (field metadata ignored). + static bool IsUnshreddedVariantType(const std::shared_ptr& type); + + /// Whether the metadata carries the Paimon Variant extension type marker. + static bool IsVariantMetadata(const std::shared_ptr& metadata); + + /// The unshredded physical Arrow type of a variant field: + /// `struct` with paimon field ids 0/1 on + /// the children. + static std::shared_ptr UnshreddedStructType(); + + /// Creates a Variant Arrow field with the variant metadata marker. + static std::shared_ptr ToArrowField( + const std::string& field_name, bool nullable = true, + std::unordered_map metadata = {}); + + /// Validates that a variant-marked field has the expected physical shape: + /// `struct`. + static Status ValidateVariantShape(const std::shared_ptr& field); + + /// Whether the schema contains a variant field, at the top level or nested inside structs, + /// arrays or maps. + static bool ContainsVariantField(const std::shared_ptr& schema); + + /// Whether the field contains a variant field, itself included, at any nesting level. + static bool ContainsVariantField(const std::shared_ptr& field); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_type_utils_test.cpp b/src/paimon/common/data/variant/variant_type_utils_test.cpp new file mode 100644 index 00000000..ba6fc4ac --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils_test.cpp @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_type_utils.h" + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(VariantTypeUtilsTest, ToArrowFieldAndDetection) { + auto field = VariantTypeUtils::ToArrowField("v"); + ASSERT_EQ(field->name(), "v"); + ASSERT_TRUE(field->nullable()); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_TRUE(VariantTypeUtils::IsVariantMetadata(field->metadata())); + ASSERT_OK(VariantTypeUtils::ValidateVariantShape(field)); + + auto struct_type = std::static_pointer_cast(field->type()); + ASSERT_EQ(struct_type->num_fields(), 2); + ASSERT_EQ(struct_type->field(0)->name(), VariantDefs::kValueFieldName); + ASSERT_EQ(struct_type->field(0)->type()->id(), arrow::Type::BINARY); + ASSERT_FALSE(struct_type->field(0)->nullable()); + ASSERT_EQ(struct_type->field(1)->name(), VariantDefs::kMetadataFieldName); + ASSERT_EQ(struct_type->field(1)->type()->id(), arrow::Type::BINARY); + ASSERT_FALSE(struct_type->field(1)->nullable()); + // The children carry paimon field ids 0/1 (mapped to parquet field ids on write). + ASSERT_EQ(struct_type->field(0)->metadata()->Get("paimon.id").ValueOr(""), "0"); + ASSERT_EQ(struct_type->field(1)->metadata()->Get("paimon.id").ValueOr(""), "1"); + + auto plain_struct = arrow::field("s", VariantTypeUtils::UnshreddedStructType()); + ASSERT_FALSE(VariantTypeUtils::IsVariantField(plain_struct)); + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto marked_binary = arrow::field("b", arrow::binary(), true, + std::make_shared(metadata)); + ASSERT_FALSE(VariantTypeUtils::IsVariantField(marked_binary)); +} + +TEST(VariantTypeUtilsTest, ValidateVariantShapeRejectsWrongShape) { + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto arrow_metadata = std::make_shared(metadata); + auto one_child = arrow::field( + "v", arrow::struct_({arrow::field("value", arrow::binary(), false)}), true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(one_child)); + auto nullable_children = + arrow::field("v", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("metadata", arrow::binary(), true)}), + true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(nullable_children)); + auto wrong_type = + arrow::field("v", + arrow::struct_({arrow::field("value", arrow::utf8(), false), + arrow::field("metadata", arrow::binary(), false)}), + true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(wrong_type)); +} + +TEST(VariantTypeUtilsTest, ContainsVariantField) { + auto variant_field = VariantTypeUtils::ToArrowField("v"); + auto plain = arrow::schema({arrow::field("a", arrow::int32())}); + ASSERT_FALSE(VariantTypeUtils::ContainsVariantField(plain)); + auto top_level = arrow::schema({arrow::field("a", arrow::int32()), variant_field}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(top_level)); + auto nested = + arrow::schema({arrow::field("row", arrow::struct_({arrow::field("inner", arrow::int32()), + VariantTypeUtils::ToArrowField("v")}))}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(nested)); + auto in_list = + arrow::schema({arrow::field("l", arrow::list(VariantTypeUtils::ToArrowField("item")))}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(in_list)); +} + +TEST(VariantTypeUtilsTest, FieldTypeConversion) { + auto variant_field = VariantTypeUtils::ToArrowField("v"); + ASSERT_OK_AND_ASSIGN(FieldType field_type, FieldTypeUtils::ConvertToFieldType(variant_field)); + ASSERT_EQ(field_type, FieldType::VARIANT); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::VARIANT), "VARIANT"); + auto plain_struct = arrow::field("s", VariantTypeUtils::UnshreddedStructType()); + ASSERT_OK_AND_ASSIGN(FieldType plain_type, FieldTypeUtils::ConvertToFieldType(plain_struct)); + ASSERT_EQ(plain_type, FieldType::STRUCT); +} + +} // namespace paimon::test diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index c025c1ae..209a08fa 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -35,7 +35,9 @@ const char Options::BUCKET_KEY[] = "bucket-key"; const char Options::FILE_FORMAT[] = "file.format"; const char Options::FILE_SYSTEM[] = "file-system"; const char Options::TARGET_FILE_SIZE[] = "target-file-size"; +const char Options::TARGET_FILE_ROW_NUM[] = "target-file-row-num"; const char Options::BLOB_TARGET_FILE_SIZE[] = "blob.target-file-size"; +const char Options::BLOB_SPLIT_BY_FILE_SIZE[] = "blob.split-by-file-size"; const char Options::PAGE_SIZE[] = "page-size"; const char Options::PARTITION_DEFAULT_NAME[] = "partition.default-name"; const char Options::FILE_COMPRESSION[] = "file.compression"; @@ -50,6 +52,8 @@ const char Options::SOURCE_SPLIT_TARGET_SIZE[] = "source.split.target-size"; const char Options::SOURCE_SPLIT_OPEN_FILE_COST[] = "source.split.open-file-cost"; const char Options::SCAN_SNAPSHOT_ID[] = "scan.snapshot-id"; const char Options::SCAN_MODE[] = "scan.mode"; +const char Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[] = + "scan.manifest-entry-cache.max-snapshots"; const char Options::READ_BATCH_SIZE[] = "read.batch-size"; const char Options::WRITE_BATCH_SIZE[] = "write.batch-size"; const char Options::WRITE_BUFFER_SIZE[] = "write-buffer-size"; @@ -66,11 +70,17 @@ const char Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES[] = "snapshot.clean-empty-d const char Options::COMMIT_FORCE_COMPACT[] = "commit.force-compact"; const char Options::COMMIT_TIMEOUT[] = "commit.timeout"; const char Options::COMMIT_MAX_RETRIES[] = "commit.max-retries"; +const char Options::COMMIT_MIN_RETRY_WAIT[] = "commit.min-retry-wait"; +const char Options::COMMIT_MAX_RETRY_WAIT[] = "commit.max-retry-wait"; +const char Options::COMMIT_DISCARD_DUPLICATE_FILES[] = "commit.discard-duplicate-files"; const char Options::SEQUENCE_FIELD[] = "sequence.field"; const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order"; const char Options::MERGE_ENGINE[] = "merge-engine"; const char Options::SORT_ENGINE[] = "sort-engine"; const char Options::IGNORE_DELETE[] = "ignore-delete"; +const char Options::FALLBACK_FIRST_ROW_IGNORE_DELETE[] = "first-row.ignore-delete"; +const char Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE[] = "deduplicate.ignore-delete"; +const char Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[] = "partial-update.ignore-delete"; const char Options::FIELDS_DEFAULT_AGG_FUNC[] = "fields.default-aggregate-function"; const char Options::DELETION_VECTORS_ENABLED[] = "deletion-vectors.enabled"; const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] = @@ -94,13 +104,33 @@ const char Options::ROW_TRACKING_PARTITION_GROUP_ON_COMMIT[] = "row-tracking.partition-group-on-commit"; const char Options::DATA_EVOLUTION_ENABLED[] = "data-evolution.enabled"; const char Options::PARTITION_GENERATE_LEGACY_NAME[] = "partition.legacy-name"; +const char Options::MAP_STORAGE_LAYOUT[] = "map.storage-layout"; +const char Options::MAP_SHARED_SHREDDING_MAX_COLUMNS[] = "map.shared-shredding.max-columns"; +const char Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[] = + "map.shared-shredding.column-placement-policy"; +const char Options::VARIANT_SHREDDING_SCHEMA[] = "variant.shreddingSchema"; +const char Options::PARQUET_VARIANT_SHREDDING_SCHEMA[] = "parquet.variant.shreddingSchema"; +const char Options::VARIANT_INFER_SHREDDING_SCHEMA[] = "variant.inferShreddingSchema"; +const char Options::VARIANT_SHREDDING_INFERENCE_MODE[] = "variant.shredding.inferenceMode"; +const char Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[] = "variant.shredding.maxSchemaWidth"; +const char Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH[] = "variant.shredding.maxSchemaDepth"; +const char Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO[] = + "variant.shredding.minFieldCardinalityRatio"; +const char Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[] = + "variant.shredding.maxInferBufferRow"; +const char Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW[] = + "variant.shredding.adaptive.maxInferBufferRow"; +const char Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO[] = + "variant.shredding.adaptive.retentionRatio"; const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor"; const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; const char Options::FALLBACK_BLOB_DESCRIPTOR_FIELD[] = "blob.stored-descriptor-fields"; const char Options::BLOB_VIEW_FIELD[] = "blob-view-field"; -const char Options::BLOB_EXTERNAL_STORAGE_FIELD[] = "blob-external-storage-field"; -const char Options::BLOB_EXTERNAL_STORAGE_PATH[] = "blob-external-storage-path"; +const char Options::BLOB_VIEW_RESOLVE_ENABLED[] = "blob-view.resolve.enabled"; +const char Options::BLOB_VIEW_UPSTREAM_WAREHOUSE[] = "blob-view-upstream-warehouse"; +const char Options::BLOB_WRITE_NULL_ON_MISSING_FILE[] = "blob-write-null-on-missing-file"; +const char Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE[] = "blob-write-null-on-fetch-failure"; const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; @@ -111,6 +141,8 @@ const char Options::SCAN_TIMESTAMP_MILLIS[] = "scan.timestamp-millis"; const char Options::SCAN_TIMESTAMP[] = "scan.timestamp"; const char Options::SCAN_TAG_NAME[] = "scan.tag-name"; const char Options::WRITE_ONLY[] = "write-only"; +const char Options::BUCKET_APPEND_ORDERED[] = "bucket-append-ordered"; +const char Options::WRITE_SEQUENCE_NUMBER_INIT_MODE[] = "write.sequence-number-init-mode"; const char Options::COMPACTION_MIN_FILE_NUM[] = "compaction.min.file-num"; const char Options::COMPACTION_FORCE_REWRITE_ALL_FILES[] = "compaction.force-rewrite-all-files"; const char Options::COMPACTION_OPTIMIZATION_INTERVAL[] = "compaction.optimization-interval"; @@ -135,6 +167,8 @@ const char Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER[] = "num-sorted-run.compa const char Options::NUM_SORTED_RUNS_STOP_TRIGGER[] = "num-sorted-run.stop-trigger"; const char Options::NUM_LEVELS[] = "num-levels"; const char Options::COMPACTION_FORCE_UP_LEVEL_0[] = "compaction.force-up-level-0"; +const char Options::OVERWRITE_UPGRADE[] = "overwrite-upgrade"; +const char Options::DYNAMIC_PARTITION_OVERWRITE[] = "dynamic-partition-overwrite"; const char Options::LOOKUP_WAIT[] = "lookup-wait"; const char Options::LOOKUP_COMPACT[] = "lookup-compact"; const char Options::LOOKUP_COMPACT_MAX_INTERVAL[] = "lookup-compact.max-interval"; diff --git a/src/paimon/common/executor/default_executor_test.cpp b/src/paimon/common/executor/default_executor_test.cpp index 2745a69b..91f2c94c 100644 --- a/src/paimon/common/executor/default_executor_test.cpp +++ b/src/paimon/common/executor/default_executor_test.cpp @@ -31,6 +31,7 @@ #include "paimon/executor.h" #include "paimon/result.h" #include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -86,7 +87,7 @@ TEST(DefaultExecutorTest, TestViaWithException) { } TEST(DefaultExecutorTest, TestShutdownNowDropsPendingTasks) { - auto executor = CreateDefaultExecutor(/*thread_count=*/1); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/1)); std::atomic first_started = false; std::atomic executed_count = 0; std::promise release_first_task; @@ -114,7 +115,7 @@ TEST(DefaultExecutorTest, TestShutdownNowDropsPendingTasks) { } TEST(DefaultExecutorTest, TestAddTaskAfterShutdownNowIgnored) { - auto executor = CreateDefaultExecutor(/*thread_count=*/1); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/1)); std::atomic executed_count = 0; executor->ShutdownNow(); @@ -125,7 +126,7 @@ TEST(DefaultExecutorTest, TestAddTaskAfterShutdownNowIgnored) { } TEST(DefaultExecutorTest, TestAddTaskFromMultipleThreads) { - auto executor = CreateDefaultExecutor(/*thread_count=*/4); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4)); constexpr int32_t kSubmitterCount = 8; constexpr int32_t kTaskCountPerSubmitter = 64; @@ -181,4 +182,9 @@ TEST(DefaultExecutorTest, TestAddTaskFromMultipleThreads) { } } +TEST(DefaultExecutorTest, TestCreateWithZeroThreadCount) { + ASSERT_NOK_WITH_MSG(CreateDefaultExecutor(/*thread_count=*/0), + "default executor thread count should be greater than 0"); +} + } // namespace paimon::test diff --git a/src/paimon/common/executor/executor.cpp b/src/paimon/common/executor/executor.cpp index f48e6d3b..cd3f699d 100644 --- a/src/paimon/common/executor/executor.cpp +++ b/src/paimon/common/executor/executor.cpp @@ -19,6 +19,7 @@ #include "paimon/executor.h" +#include #include #include #include @@ -35,14 +36,14 @@ class DefaultExecutor : public Executor { ~DefaultExecutor() override; void Add(std::function func) override; - void ShutdownNow() override; + uint32_t GetThreadNum() const override; private: void WorkerThread(); - void ShutdownInternal(bool wait_for_pending_tasks); + private: uint32_t thread_count_; std::vector workers_; std::queue> tasks_; @@ -53,11 +54,16 @@ class DefaultExecutor : public Executor { }; DefaultExecutor::DefaultExecutor(uint32_t thread_count) : thread_count_(thread_count) { + assert(thread_count > 0); for (uint32_t i = 0; i < thread_count_; ++i) { workers_.emplace_back(&DefaultExecutor::WorkerThread, this); } } +uint32_t DefaultExecutor::GetThreadNum() const { + return thread_count_; +} + void DefaultExecutor::ShutdownInternal(bool wait_for_pending_tasks) { { std::unique_lock lock(queue_mutex_); @@ -133,15 +139,18 @@ void DefaultExecutor::WorkerThread() { PAIMON_EXPORT std::shared_ptr GetGlobalDefaultExecutor() { static uint32_t all_cores = std::thread::hardware_concurrency(); static std::shared_ptr internal = - std::make_shared(/*thread_count=*/all_cores); + std::make_shared(/*thread_count=*/all_cores > 0 ? all_cores : 1); return internal; } PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor() { - return CreateDefaultExecutor(DEFAULT_EXECUTOR_THREAD_COUNT); + return std::make_unique(DEFAULT_EXECUTOR_THREAD_COUNT); } -PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(uint32_t thread_count) { +PAIMON_EXPORT Result> CreateDefaultExecutor(uint32_t thread_count) { + if (thread_count == 0) { + return Status::Invalid("default executor thread count should be greater than 0"); + } return std::make_unique(thread_count); } diff --git a/src/paimon/common/file_index/CMakeLists.txt b/src/paimon/common/file_index/CMakeLists.txt index 9a1419e3..7ab5a069 100644 --- a/src/paimon/common/file_index/CMakeLists.txt +++ b/src/paimon/common/file_index/CMakeLists.txt @@ -44,7 +44,6 @@ add_paimon_lib(paimon_file_index arrow fmt xxhash - dl Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h index 28957512..4894ad06 100644 --- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h +++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h @@ -82,8 +82,8 @@ class ApplyBitmapIndexBatchReader : public FileBatchReader { return Status::Invalid("ApplyBitmapIndexBatchReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -96,14 +96,23 @@ class ApplyBitmapIndexBatchReader : public FileBatchReader { private: Result Filter(int32_t batch_size) const { - RoaringBitmap32 is_valid; - PAIMON_ASSIGN_OR_RAISE(int32_t start_pos, reader_->GetPreviousBatchFirstRowNumber()); - int32_t length = batch_size; - for (auto iter = bitmap_.EqualOrLarger(start_pos); - iter != bitmap_.End() && *iter < start_pos + length; ++iter) { - is_valid.Add(*iter - start_pos); + RoaringBitmap32 result; + auto bitmap_iter = bitmap_.Begin(); + auto bitmap_end = bitmap_.End(); + + for (int32_t i = 0; i < batch_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, reader_->GetPreviousBatchFileRowId(i)); + while (bitmap_iter != bitmap_end && static_cast(*bitmap_iter) < file_row_id) { + ++bitmap_iter; + } + if (bitmap_iter == bitmap_end) { + break; + } + if (static_cast(*bitmap_iter) == file_row_id) { + result.Add(i); + } } - return is_valid; + return result; } private: diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp index 1168a358..7b1ebd3f 100644 --- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp +++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp @@ -56,7 +56,7 @@ class ApplyBitmapIndexBatchReaderTest : public ::testing::Test, pool_ = GetDefaultPool(); fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); } void TearDown() override {} diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp index cbe92a8b..eccbb1f3 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp @@ -625,7 +625,7 @@ TEST_F(BitmapIndexTest, TestHighCardinalityForCompatibility) { auto file_system = std::make_unique(); ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(index_file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); BitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN(auto reader, file_index.CreateReader(CreateArrowSchema(type).get(), diff --git a/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp b/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp index 1027ce52..809ceeae 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp @@ -50,7 +50,7 @@ Result> BitmapIndexResult::And( typed_other]() -> Result { PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, result->GetBitmap()); PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* other_bitmap, - typed_other->GetBitmap()) + typed_other->GetBitmap()); return RoaringBitmap32::And(*bitmap, *other_bitmap); }); } @@ -66,7 +66,7 @@ Result> BitmapIndexResult::Or( typed_other]() -> Result { PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, result->GetBitmap()); PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* other_bitmap, - typed_other->GetBitmap()) + typed_other->GetBitmap()); return RoaringBitmap32::Or(*bitmap, *other_bitmap); }); } diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp index b0a40ff0..ca33c696 100644 --- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp +++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp @@ -47,7 +47,7 @@ Result> BloomFilterFileIndex::CreateReader( PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET)); auto bytes = std::make_shared(length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, input_stream->Read(bytes->data(), bytes->size())); if (static_cast(actual_read_len) != bytes->size()) { return Status::Invalid( @@ -80,9 +80,15 @@ BloomFilterFileIndexReader::BloomFilterFileIndexReader(const FastHash::HashFunct Result> BloomFilterFileIndexReader::VisitEqual( const Literal& literal) { + // This returns `Remain` to align with the current Java implementation in BF index, even though + // its predicate semantics are inconsistent here. In practice, equality tests in predicate + // evaluation always return false when the literal is null. See + // `null_false_leaf_binary_function.h`. + if (literal.IsNull()) { + return FileIndexResult::Remain(); + } int64_t hash = hash_function_(literal); - return literal.IsNull() || filter_.TestHash(hash) ? FileIndexResult::Remain() - : FileIndexResult::Skip(); + return filter_.TestHash(hash) ? FileIndexResult::Remain() : FileIndexResult::Skip(); } } // namespace paimon diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp index 49ab89ea..aca978bd 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp @@ -19,7 +19,9 @@ #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h" #include +#include #include +#include #include "fmt/format.h" #include "paimon/common/file_index/bsi/bit_slice_index_roaring_bitmap.h" @@ -33,7 +35,17 @@ #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap32.h" +namespace { +// Safe absolute value for int64_t that avoids undefined behavior when value == INT64_MIN. +// This mirrors Java's Math.abs() wrapping semantics. +inline int64_t SafeAbs(int64_t value) { + if (value == INT64_MIN) { + return INT64_MIN; + } + return value < 0 ? -value : value; +} +} // namespace namespace paimon { class MemoryPool; @@ -55,7 +67,7 @@ Result> BitSliceIndexBitmapFileIndex::CreateRea PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET)); auto bytes = std::make_unique(length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, input_stream->Read(bytes->data(), bytes->size())); if (static_cast(actual_read_len) != bytes->size()) { return Status::Invalid( @@ -155,10 +167,14 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value >= 0) { + if (value == INT64_MIN) { + // Everything is greater than INT64_MIN (writer cannot store it) + return RoaringBitmap32::Or(reader->positive_->IsNotNull(), + reader->negative_->IsNotNull()); + } else if (value >= 0) { return reader->positive_->GreaterThan(value); } else { - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessThan(-value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessThan(SafeAbs(value))); RoaringBitmap32 b2 = reader->positive_->IsNotNull(); b1 |= b2; return b1; @@ -172,10 +188,15 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value >= 0) { + if (value == INT64_MIN) { + // All non-null rows satisfy x >= INT64_MIN + return RoaringBitmap32::Or(reader->positive_->IsNotNull(), + reader->negative_->IsNotNull()); + } else if (value >= 0) { return reader->positive_->GreaterOrEqual(value); } else { - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessOrEqual(-value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, + reader->negative_->LessOrEqual(SafeAbs(value))); RoaringBitmap32 b2 = reader->positive_->IsNotNull(); b1 |= b2; return b1; @@ -189,8 +210,11 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value < 0) { - return reader->negative_->GreaterThan(-value); + if (value == INT64_MIN) { + // Nothing is less than INT64_MIN + return RoaringBitmap32(); + } else if (value < 0) { + return reader->negative_->GreaterThan(SafeAbs(value)); } else { PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->positive_->LessThan(value)); RoaringBitmap32 b2 = reader->negative_->IsNotNull(); @@ -205,8 +229,11 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value < 0) { - return reader->negative_->GreaterOrEqual(-value); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match + return RoaringBitmap32(); + } else if (value < 0) { + return reader->negative_->GreaterOrEqual(SafeAbs(value)); } else { PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->positive_->LessOrEqual(value)); RoaringBitmap32 b2 = reader->negative_->IsNotNull(); @@ -234,13 +261,17 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis result_bitmaps.reserve(literals.size()); for (const auto& literal : literals) { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - RoaringBitmap32 equal; - if (value < 0) { - PAIMON_ASSIGN_OR_RAISE(equal, reader->negative_->Equal(-value)); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match it + result_bitmaps.emplace_back(); + } else if (value < 0) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, + reader->negative_->Equal(SafeAbs(value))); + result_bitmaps.emplace_back(std::move(equal)); } else { - PAIMON_ASSIGN_OR_RAISE(equal, reader->positive_->Equal(value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, reader->positive_->Equal(value)); + result_bitmaps.emplace_back(std::move(equal)); } - result_bitmaps.emplace_back(std::move(equal)); } return RoaringBitmap32::FastUnion(result_bitmaps); }; @@ -257,13 +288,17 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis result_bitmaps.reserve(literals.size()); for (const auto& literal : literals) { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - RoaringBitmap32 equal; - if (value < 0) { - PAIMON_ASSIGN_OR_RAISE(equal, reader->negative_->Equal(-value)); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match it + result_bitmaps.emplace_back(); + } else if (value < 0) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, + reader->negative_->Equal(SafeAbs(value))); + result_bitmaps.emplace_back(std::move(equal)); } else { - PAIMON_ASSIGN_OR_RAISE(equal, reader->positive_->Equal(value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, reader->positive_->Equal(value)); + result_bitmaps.emplace_back(std::move(equal)); } - result_bitmaps.emplace_back(std::move(equal)); } auto in = RoaringBitmap32::FastUnion(result_bitmaps); ebm -= in; diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp index b678ba3f..128c1a69 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp @@ -18,6 +18,7 @@ #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h" +#include #include #include "gtest/gtest.h" @@ -387,4 +388,49 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestUnInvalidType) { "BitSliceIndexBitmapFileIndex only support TINYINT/SMALLINT/INT/BIGINT/DATE"); } +TEST_F(BitSliceIndexBitmapIndexReaderTest, TestReaderPredicatePruningWithInt64Min) { + // Reuse TestPrimitiveType's index bytes. + // data: null, 1, null, 2, -1 (non-null rows: {1, 3, 4}) + std::vector index_bytes = { + 1, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 2, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, + 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0}; + auto input_stream = + std::make_shared(index_bytes.data(), index_bytes.size()); + BitSliceIndexBitmapFileIndex file_index({}); + ASSERT_OK_AND_ASSIGN( + auto reader, + file_index.CreateReader(CreateArrowSchema(arrow::int64()).get(), + /*start=*/0, /*length=*/index_bytes.size(), input_stream, pool_)); + + // x > INT64_MIN: all non-null rows (writer cannot store INT64_MIN) + CheckResult(reader->VisitGreaterThan(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x >= INT64_MIN: all non-null rows + CheckResult(reader->VisitGreaterOrEqual(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x < INT64_MIN: empty + CheckResult(reader->VisitLessThan(Literal(INT64_MIN)).value(), {}); + + // x <= INT64_MIN: empty (no row has INT64_MIN) + CheckResult(reader->VisitLessOrEqual(Literal(INT64_MIN)).value(), {}); + + // x == INT64_MIN: empty + CheckResult(reader->VisitEqual(Literal(INT64_MIN)).value(), {}); + + // x != INT64_MIN: all non-null rows + CheckResult(reader->VisitNotEqual(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x IN (INT64_MIN, -1): only -1 matches → row 4 + CheckResult(reader->VisitIn({Literal(INT64_MIN), Literal(static_cast(-1))}).value(), + {4}); + + // x NOT IN (INT64_MIN, -1): all non-null except row 4 + CheckResult(reader->VisitNotIn({Literal(INT64_MIN), Literal(static_cast(-1))}).value(), + {1, 3}); +} + } // namespace paimon::test diff --git a/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp b/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp index 18f71076..5743bebc 100644 --- a/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp +++ b/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "paimon/common/io/memory_segment_output_stream.h" diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp index 51da76a7..e2bb867e 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp @@ -105,7 +105,7 @@ TEST_F(RangeBitmapIoTest, TestSimple) { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/false)); ASSERT_OK_AND_ASSIGN( - int32_t write_len, + int64_t write_len, out->Write(reinterpret_cast(serialized_bytes->data()), serialized_bytes->size())); ASSERT_EQ(write_len, serialized_bytes->size()); ASSERT_OK(out->Flush()); diff --git a/src/paimon/common/fs/file_system.cpp b/src/paimon/common/fs/file_system.cpp index 23895fc1..a5ae3ce8 100644 --- a/src/paimon/common/fs/file_system.cpp +++ b/src/paimon/common/fs/file_system.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -50,10 +51,11 @@ Status FileSystem::ReadFile(const std::string& path, std::string* content) { Status s = in->Close(); (void)s; }); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, in->Length()); - content->resize(length); - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, in->Read(content->data(), length)); - if (read_length != static_cast(length)) { + PAIMON_ASSIGN_OR_RAISE(int64_t length, in->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); + content->resize(static_cast(length)); + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, in->Read(content->data(), length)); + if (read_length != length) { return Status::IOError(fmt::format("path {}, expect read len {}, actual read len {}", path, length, read_length)); } @@ -71,8 +73,8 @@ Status FileSystem::WriteFile(const std::string& path, const std::string& content Status s = out->Close(); (void)s; }); - int32_t length = content.size(); - PAIMON_ASSIGN_OR_RAISE(int32_t write_length, out->Write(content.data(), length)); + auto length = static_cast(content.size()); + PAIMON_ASSIGN_OR_RAISE(int64_t write_length, out->Write(content.data(), length)); if (write_length != length) { return Status::IOError(fmt::format("path {}, expect write len {}, actual write len {}", path, length, write_length)); diff --git a/src/paimon/common/fs/file_system_test.cpp b/src/paimon/common/fs/file_system_test.cpp index b7188c4b..49482381 100644 --- a/src/paimon/common/fs/file_system_test.cpp +++ b/src/paimon/common/fs/file_system_test.cpp @@ -94,7 +94,7 @@ class FileSystemTest : public ::testing::Test, public ::testing::WithParamInterf fs_->Create(file, /*overwrite=*/true)); std::string input = "paimon"; char chars[8] = {1, 2, 3, 4, 5, 6, 7, 8}; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(chars, input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(chars, input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Flush()); ASSERT_OK(out->Close()); @@ -210,7 +210,7 @@ TEST_P(FileSystemTest, TestCreate) { ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); ASSERT_TRUE(out); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Close()); @@ -225,7 +225,7 @@ TEST_P(FileSystemTest, TestCreateRelativeFileInCurrentDirectory) { std::string path = "relative_file_" + RandomName(); ASSERT_OK_AND_ASSIGN(auto out, fs_->Create(path, /*overwrite=*/true)); std::string content = "content"; - ASSERT_OK_AND_ASSIGN(int32_t write_len, out->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK_AND_ASSIGN(std::string uri, out->GetUri()); ASSERT_FALSE(uri.empty()); @@ -245,7 +245,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); @@ -263,7 +263,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { // read from cur pos std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -275,7 +275,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { ASSERT_OK_AND_ASSIGN(uri, in_stream->GetUri()); ASSERT_EQ(uri, file_path); - ASSERT_OK_AND_ASSIGN(uint64_t file_len, in_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_len, in_stream->Length()); ASSERT_EQ(file_len, content.size()); ASSERT_OK_AND_ASSIGN(pos, in_stream->GetPos()); @@ -290,7 +290,7 @@ TEST_P(FileSystemTest, TestWriteMultipleTimes) { // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); for (const auto& str : content_vec) { - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(str.data(), str.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(str.data(), str.size())); ASSERT_EQ(write_len, str.size()); } ASSERT_OK(out_stream->Flush()); @@ -301,7 +301,7 @@ TEST_P(FileSystemTest, TestWriteMultipleTimes) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -312,7 +312,7 @@ TEST_P(FileSystemTest, TestWriteInNotExistDir) { // write process std::string content = "abcdefghijk"; ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -320,7 +320,7 @@ TEST_P(FileSystemTest, TestWriteInNotExistDir) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -334,7 +334,7 @@ TEST_P(FileSystemTest, TestWriteEmptyFile) { // write process std::string content = ""; ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, 0); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -351,7 +351,7 @@ TEST_P(FileSystemTest, TestWriteEmptyFile) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -362,7 +362,7 @@ TEST_P(FileSystemTest, TestWriteWithOverwrite) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -379,7 +379,7 @@ TEST_P(FileSystemTest, TestWriteWithOverwrite) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(new_content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(new_content, read_content); @@ -395,7 +395,7 @@ TEST_P(FileSystemTest, TestAsyncRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -428,7 +428,7 @@ TEST_P(FileSystemTest, TestInvalidRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -441,7 +441,7 @@ TEST_P(FileSystemTest, TestInvalidRead) { // read from cur pos std::string read_content(3, '\0'); ASSERT_NOK(in_stream->Read(read_content.data(), read_content.size())); - ASSERT_OK_AND_ASSIGN(size_t actual_read, in_stream->Read(read_content.data(), 0)); + ASSERT_OK_AND_ASSIGN(int64_t actual_read, in_stream->Read(read_content.data(), 0)); ASSERT_EQ(actual_read, 0); } { @@ -465,7 +465,7 @@ TEST_P(FileSystemTest, TestInvalidAsyncRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -567,7 +567,7 @@ TEST_P(FileSystemTest, TestSeek) { std::string path = PathUtil::JoinPath(test_root_, "/test_file"); ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Close()); @@ -585,7 +585,7 @@ TEST_P(FileSystemTest, TestSeek) { ASSERT_EQ(pos3, 1); std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, in->Read(read_content.data(), read_content.size())); + ASSERT_OK_AND_ASSIGN(int64_t read_len, in->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("aim", read_content); @@ -604,7 +604,7 @@ TEST_P(FileSystemTest, TestSeek2) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -629,7 +629,7 @@ TEST_P(FileSystemTest, TestSeek2) { // read from cur pos std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("ijk", read_content); @@ -651,14 +651,14 @@ TEST_P(FileSystemTest, TestRename) { ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); ASSERT_TRUE(out); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Flush()); ASSERT_OK(out->Close()); ASSERT_OK_AND_ASSIGN(std::unique_ptr in, fs_->Open(path)); ASSERT_TRUE(in); char* data = new char[input.size() * 2]; - ASSERT_OK_AND_ASSIGN(int32_t size_read, in->Read(data, input.size(), /*offset=*/0)); + ASSERT_OK_AND_ASSIGN(int64_t size_read, in->Read(data, input.size(), /*offset=*/0)); ASSERT_EQ(size_read, input.size()); std::string read_data(data, input.size()); ASSERT_EQ(read_data, input); @@ -860,7 +860,8 @@ TEST_P(FileSystemTest, TestExistingFileDeletion) { TEST_P(FileSystemTest, TestNotExistingFileDeletion) { auto check = [&](bool recursive) { std::string path = PathUtil::JoinPath(test_root_, RandomName()); - ASSERT_TRUE(fs_->Delete(path, recursive).IsIOError()); + Status status = fs_->Delete(path, recursive); + ASSERT_TRUE(status.IsIOError() || status.IsNotExist()) << status.ToString(); }; check(true); check(false); @@ -1089,7 +1090,7 @@ TEST_P(FileSystemTest, TestMkdir2) { TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNonExistParentDir) { uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; @@ -1114,7 +1115,7 @@ TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNonExistParentDir) { TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameName) { uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; @@ -1134,9 +1135,12 @@ TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameName) { // test for create multi dir such as "partition1" and "partition1" (relative path) TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNameWithRelativePath) { + if (GetParam() == "jindo") { + GTEST_SKIP() << "skip jindo for relative path test"; + } uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; @@ -1474,7 +1478,9 @@ TEST_P(FileSystemTest, TestAtomicStoreAlreadyExist) { std::vector GetTestValuesForFileSystemTest() { std::vector values; values.emplace_back("local"); - // values.emplace_back("jindo"); +#if defined(PAIMON_ENABLE_NETWORK_TESTS) && defined(PAIMON_ENABLE_JINDO) + values.emplace_back("jindo"); +#endif return values; } diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/fs/http_client.cpp new file mode 100644 index 00000000..3ff589b4 --- /dev/null +++ b/src/paimon/common/fs/http_client.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/fs/http_client.h" + +#include + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +namespace { + +constexpr int32_t kMaxAttempts = 3; + +class CurlGlobalGuard { + public: + CurlGlobalGuard() { + curl_global_init(CURL_GLOBAL_DEFAULT); + } + ~CurlGlobalGuard() { + curl_global_cleanup(); + } +}; + +std::shared_ptr GetCurlGlobalGuard() { + static auto guard = std::make_shared(); + return guard; +} + +void TrimHttpWhitespace(std::string* value) { + constexpr char kHttpWhitespace[] = " \t\r\n"; + size_t begin = value->find_first_not_of(kHttpWhitespace); + if (begin == std::string::npos) { + value->clear(); + return; + } + size_t end = value->find_last_not_of(kHttpWhitespace); + value->erase(end + 1); + value->erase(0, begin); +} + +struct TransferContext { + CURL* handle; + const HttpBodyConsumer* consumer; + HttpResponse response; + Status status = Status::OK(); +}; + +size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) { + auto* context = static_cast(user_data); + size_t bytes = size * count; + if (!context->status.ok()) { + return 0; + } + long status_code = 0; // NOLINT(runtime/int, google-runtime-int): required by curl. + curl_easy_getinfo(context->handle, CURLINFO_RESPONSE_CODE, &status_code); + context->response.status_code = static_cast(status_code); + if (status_code >= 300) { + context->response.body_size += static_cast(bytes); + return bytes; + } + context->status = (*context->consumer)(data, static_cast(bytes)); + if (!context->status.ok()) { + return 0; + } + context->response.body_size += static_cast(bytes); + return bytes; +} + +size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) { + auto* context = static_cast(user_data); + size_t bytes = size * count; + std::string line(data, bytes); + size_t colon = line.find(':'); + if (colon != std::string::npos) { + std::string name = line.substr(0, colon); + TrimHttpWhitespace(&name); + name = StringUtils::ToLowerCase(name); + std::string value = line.substr(colon + 1); + TrimHttpWhitespace(&value); + context->response.headers[name] = std::move(value); + } + return bytes; +} + +bool IsRetryable(CURLcode code, int64_t status_code) { + if (code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_COULDNT_CONNECT || + code == CURLE_OPERATION_TIMEDOUT || code == CURLE_SEND_ERROR || code == CURLE_RECV_ERROR || + code == CURLE_PARTIAL_FILE) { + return true; + } + return status_code == 429 || status_code >= 500; +} + +} // namespace + +class CurlHttpClient::Impl { + public: + Impl() : guard_(GetCurlGlobalGuard()) {} + + ~Impl() { + for (CURL* handle : handles_) { + curl_easy_cleanup(handle); + } + } + + CURL* Acquire() const { + std::scoped_lock lock(mutex_); + if (handles_.empty()) { + return curl_easy_init(); + } + CURL* handle = handles_.back(); + handles_.pop_back(); + return handle; + } + + void Release(CURL* handle) const { + curl_easy_reset(handle); + std::scoped_lock lock(mutex_); + handles_.push_back(handle); + } + + private: + std::shared_ptr guard_; + mutable std::mutex mutex_; + mutable std::vector handles_; +}; + +CurlHttpClient::CurlHttpClient() : impl_(std::make_unique()) {} +CurlHttpClient::~CurlHttpClient() = default; + +Result CurlHttpClient::Execute(const HttpRequest& request, + const HttpBodyConsumer& consumer) const { + for (int32_t attempt = 0; attempt < kMaxAttempts; ++attempt) { + CURL* handle = impl_->Acquire(); + if (handle == nullptr) { + return Status::IOError("failed to create curl easy handle"); + } + ScopeGuard release_handle([this, handle] { impl_->Release(handle); }); + TransferContext context{handle, &consumer, {}, Status::OK()}; + curl_slist* headers = nullptr; + ScopeGuard release_headers([&headers] { curl_slist_free_all(headers); }); + for (const auto& [name, value] : request.headers) { + curl_slist* updated_headers = curl_slist_append(headers, (name + ": " + value).c_str()); + if (updated_headers == nullptr) { + return Status::IOError("failed to create HTTP headers"); + } + headers = updated_headers; + } + curl_easy_setopt(handle, CURLOPT_URL, request.url.c_str()); + curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 30000L); + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(handle, CURLOPT_WRITEDATA, &context); + curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, HeaderCallback); + curl_easy_setopt(handle, CURLOPT_HEADERDATA, &context); + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 0L); + if (request.method == HttpMethod::HEAD) { + curl_easy_setopt(handle, CURLOPT_NOBODY, 1L); + } + CURLcode code = curl_easy_perform(handle); + long response_code = 0; // NOLINT(runtime/int, google-runtime-int): required by curl. + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code); + context.response.status_code = static_cast(response_code); + + if (!context.status.ok()) { + return context.status; + } + if (code == CURLE_OK && !IsRetryable(code, response_code)) { + return context.response; + } + if (!IsRetryable(code, response_code) || + (context.response.body_size > 0 && response_code < 300) || + attempt + 1 == kMaxAttempts) { + if (code == CURLE_OK) { + return Status::IOError(fmt::format("HTTP request to {} returned status {}", + request.url, response_code)); + } + return Status::IOError(fmt::format("HTTP request to {} failed: {} (status {})", + request.url, curl_easy_strerror(code), + response_code)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100 * (1 << attempt))); + } + return Status::IOError("HTTP request failed"); +} + +} // namespace paimon diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/fs/http_client.h new file mode 100644 index 00000000..5dd8d029 --- /dev/null +++ b/src/paimon/common/fs/http_client.h @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +enum class HttpMethod { HEAD, GET }; +using HttpHeaders = std::map; +using HttpBodyConsumer = std::function; + +struct HttpRequest { + HttpMethod method = HttpMethod::GET; + std::string url; + HttpHeaders headers; +}; + +struct HttpResponse { + int32_t status_code = 0; + HttpHeaders headers; + int64_t body_size = 0; +}; + +class PAIMON_EXPORT HttpClient { + public: + virtual ~HttpClient() = default; + virtual Result Execute(const HttpRequest& request, + const HttpBodyConsumer& consumer) const = 0; +}; + +class PAIMON_EXPORT CurlHttpClient : public HttpClient { + public: + CurlHttpClient(); + ~CurlHttpClient() override; + + Result Execute(const HttpRequest& request, + const HttpBodyConsumer& consumer) const override; + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace paimon diff --git a/src/paimon/common/fs/object_store_file_system.cpp b/src/paimon/common/fs/object_store_file_system.cpp new file mode 100644 index 00000000..2eb2b9be --- /dev/null +++ b/src/paimon/common/fs/object_store_file_system.cpp @@ -0,0 +1,535 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/fs/object_store_file_system.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/math.h" +#include "paimon/common/utils/path_util.h" + +namespace paimon { +namespace { + +constexpr int64_t kMaxReadAheadMemory = 64LL * 1024LL * 1024LL; +constexpr int64_t kInitialReadAheadSize = 64LL * 1024LL; +constexpr int64_t kMaxReadAheadSize = 8LL * 1024LL * 1024LL; + +std::string NormalizeDirectoryPrefix(const std::string& key) { + if (key.empty() || key.back() == '/') { + return key; + } + return key + "/"; +} + +class ObjectStoreBasicFileStatus : public BasicFileStatus { + public: + ObjectStoreBasicFileStatus(std::string path, bool is_dir) + : path_(std::move(path)), is_dir_(is_dir) {} + bool IsDir() const override { + return is_dir_; + } + std::string GetPath() const override { + return path_; + } + + private: + std::string path_; + bool is_dir_; +}; + +class ObjectStoreFileStatus : public FileStatus { + public: + ObjectStoreFileStatus(std::string path, int64_t size, int64_t modification_time, bool is_dir) + : path_(std::move(path)), + size_(size), + modification_time_(modification_time), + is_dir_(is_dir) {} + int64_t GetLen() const override { + return size_; + } + bool IsDir() const override { + return is_dir_; + } + std::string GetPath() const override { + return path_; + } + int64_t GetModificationTime() const override { + return modification_time_; + } + + private: + std::string path_; + int64_t size_; + int64_t modification_time_; + bool is_dir_; +}; + +class ObjectStoreInputStream : public InputStream { + public: + ObjectStoreInputStream(std::shared_ptr client, + std::shared_ptr limiter, ObjectStorePath path, + std::string uri, int64_t length) + : client_(std::move(client)), + limiter_(std::move(limiter)), + path_(std::move(path)), + uri_(std::move(uri)), + length_(length) {} + + ~ObjectStoreInputStream() override { + CloseInternal(); + } + + Status Seek(int64_t offset, SeekOrigin origin) override { + std::scoped_lock lock(mutex_); + if (closed_) { + return Status::IOError(fmt::format("{} is closed", uri_)); + } + int64_t base; + if (origin == FS_SEEK_SET) { + base = 0; + } else if (origin == FS_SEEK_CUR) { + base = position_; + } else if (origin == FS_SEEK_END) { + base = length_; + } else { + return Status::Invalid("unsupported seek origin"); + } + if ((offset > 0 && base > std::numeric_limits::max() - offset) || + (offset < 0 && base < std::numeric_limits::min() - offset)) { + return Status::Invalid("object store input position overflows"); + } + int64_t position = base + offset; + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(position, "object store input position")); + position_ = position; + return Status::OK(); + } + + Result GetPos() const override { + std::scoped_lock lock(mutex_); + if (closed_) { + return Status::IOError(fmt::format("{} is closed", uri_)); + } + return position_; + } + + Result Read(char* buffer, int64_t size) override { + std::scoped_lock read_lock(read_mutex_); + int64_t offset; + { + std::scoped_lock lock(mutex_); + if (closed_) { + return Status::IOError(fmt::format("{} is closed", uri_)); + } + offset = position_; + } + PAIMON_ASSIGN_OR_RAISE(int64_t bytes_read, ReadWithReadAhead(buffer, size, offset)); + { + std::scoped_lock lock(mutex_); + position_ += bytes_read; + } + return bytes_read; + } + + Result Read(char* buffer, int64_t size, int64_t offset) override { + std::scoped_lock read_lock(read_mutex_); + return ReadWithReadAhead(buffer, size, offset); + } + + void ReadAsync(char* buffer, int64_t size, int64_t offset, + std::function&& callback) override { + Status status = ValidateRead(buffer, size, offset); + if (!status.ok()) { + callback(std::move(status)); + return; + } + if (offset > length_ || size > length_ - offset) { + callback(Status::Invalid( + fmt::format("object store async read size {} at offset {} exceeds length {}", size, + offset, length_))); + return; + } + if (size == 0) { + callback(Status::OK()); + return; + } + client_->GetObjectRangeAsync(path_, offset, size, buffer, + [callback = std::move(callback)](Status status) mutable { + if (!status.ok()) { + callback(std::move(status)); + return; + } + callback(Status::OK()); + }); + } + + Status Close() override { + CloseInternal(); + return Status::OK(); + } + Result GetUri() const override { + return uri_; + } + Result Length() const override { + return length_; + } + + private: + Status ValidateRead(char* buffer, int64_t size, int64_t offset) const { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); + { + std::scoped_lock lock(mutex_); + if (closed_) { + return Status::IOError(fmt::format("{} is closed", uri_)); + } + } + if (size > 0 && buffer == nullptr) { + return Status::Invalid("read buffer is null"); + } + return Status::OK(); + } + + void EvictReadAhead() { + int64_t reserved = 0; + { + std::scoped_lock lock(mutex_); + std::vector().swap(read_ahead_buffer_); + reserved = reserved_read_ahead_size_; + reserved_read_ahead_size_ = 0; + } + limiter_->Release(reserved); + } + + Result ReadWithReadAhead(char* buffer, int64_t size, int64_t offset) { + PAIMON_RETURN_NOT_OK(ValidateRead(buffer, size, offset)); + if (size == 0 || offset >= length_) { + return 0; + } + int64_t read_size = std::min(size, length_ - offset); + bool continues_previous = false; + { + std::scoped_lock lock(mutex_); + int64_t relative = offset - read_ahead_offset_; + if (relative >= 0 && relative <= static_cast(read_ahead_buffer_.size()) && + read_size <= static_cast(read_ahead_buffer_.size()) - relative) { + std::memcpy(buffer, read_ahead_buffer_.data() + relative, + static_cast(read_size)); + return read_size; + } + continues_previous = + !read_ahead_buffer_.empty() && + offset == read_ahead_offset_ + static_cast(read_ahead_buffer_.size()); + } + EvictReadAhead(); + if (read_size >= kMaxReadAheadSize) { + next_read_ahead_size_ = kInitialReadAheadSize; + return client_->GetObjectRange(path_, offset, read_size, buffer); + } + next_read_ahead_size_ = continues_previous + ? std::min(kMaxReadAheadSize, next_read_ahead_size_ * 2) + : kInitialReadAheadSize; + int64_t fetch_size = std::min(std::max(read_size, next_read_ahead_size_), length_ - offset); + fetch_size = limiter_->ReserveUpTo(read_size, fetch_size); + if (fetch_size == 0) { + return client_->GetObjectRange(path_, offset, read_size, buffer); + } + std::vector fetched(static_cast(fetch_size)); + Result result = client_->GetObjectRange(path_, offset, fetch_size, fetched.data()); + if (!result.ok()) { + limiter_->Release(fetch_size); + return result.status(); + } + int64_t bytes_read = std::move(result).value(); + if (bytes_read != fetch_size) { + limiter_->Release(fetch_size); + return Status::IOError(fmt::format("range read for {} returned {} bytes, expected {}", + uri_, bytes_read, fetch_size)); + } + { + std::scoped_lock lock(mutex_); + if (closed_) { + limiter_->Release(fetch_size); + return Status::IOError(fmt::format("{} is closed", uri_)); + } + read_ahead_offset_ = offset; + read_ahead_buffer_ = std::move(fetched); + reserved_read_ahead_size_ = fetch_size; + std::memcpy(buffer, read_ahead_buffer_.data(), static_cast(read_size)); + } + return read_size; + } + + void CloseInternal() { + int64_t reserved = 0; + { + std::scoped_lock lock(mutex_); + closed_ = true; + std::vector().swap(read_ahead_buffer_); + reserved = reserved_read_ahead_size_; + reserved_read_ahead_size_ = 0; + } + limiter_->Release(reserved); + } + + std::shared_ptr client_; + std::shared_ptr limiter_; + ObjectStorePath path_; + std::string uri_; + int64_t length_; + int64_t position_ = 0; + int64_t read_ahead_offset_ = 0; + int64_t next_read_ahead_size_ = kInitialReadAheadSize; + int64_t reserved_read_ahead_size_ = 0; + std::vector read_ahead_buffer_; + bool closed_ = false; + mutable std::mutex read_mutex_; + mutable std::mutex mutex_; +}; + +} // namespace + +ReadAheadMemoryLimiter::ReadAheadMemoryLimiter(int64_t limit) : limit_(limit) {} + +int64_t ReadAheadMemoryLimiter::ReserveUpTo(int64_t min_size, int64_t max_size) { + std::scoped_lock lock(mutex_); + int64_t available = limit_ - used_; + if (available < min_size) { + return 0; + } + int64_t size = std::min(max_size, available); + used_ += size; + return size; +} + +void ReadAheadMemoryLimiter::Release(int64_t size) { + std::scoped_lock lock(mutex_); + assert(size >= 0 && size <= used_); + used_ -= std::clamp(size, int64_t{0}, used_); +} + +ObjectStoreFileSystem::ObjectStoreFileSystem(std::string scheme, + std::shared_ptr client) + : ObjectStoreFileSystem(std::move(scheme), std::move(client), kMaxReadAheadMemory) {} + +ObjectStoreFileSystem::ObjectStoreFileSystem(std::string scheme, + std::shared_ptr client, + int64_t read_ahead_memory_limit) + : scheme_(std::move(scheme)), + client_(std::move(client)), + read_ahead_limiter_(std::make_shared(read_ahead_memory_limit)) {} + +Result ObjectStoreFileSystem::ParsePath(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(Path parsed, PathUtil::ToPath(path)); + if (parsed.scheme != scheme_) { + return Status::Invalid(fmt::format("path must use {} scheme: {}", scheme_, path)); + } + if (parsed.authority.empty()) { + return Status::Invalid(fmt::format("{} path must include bucket: {}", scheme_, path)); + } + std::string key = parsed.path; + key.erase(0, key.find_first_not_of('/')); + return ObjectStorePath{parsed.authority, key}; +} + +std::string ObjectStoreFileSystem::ToUri(const ObjectStorePath& path, bool is_dir) const { + std::string uri = scheme_ + "://" + path.bucket + "/"; + uri += path.key; + if (is_dir && uri.back() != '/') { + uri += "/"; + } + return uri; +} + +Result ObjectStoreFileSystem::DirectoryExists(const ObjectStorePath& path) const { + ObjectStorePath directory{path.bucket, NormalizeDirectoryPrefix(path.key)}; + PAIMON_ASSIGN_OR_RAISE(ListObjectsResult result, client_->ListObjects(directory, "", 1)); + return path.key.empty() || !result.objects.empty() || !result.common_prefixes.empty(); +} + +Result> ObjectStoreFileSystem::Open(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); + if (object_path.key.empty()) { + return Status::Invalid(fmt::format("{} is a directory", path)); + } + Result metadata = client_->HeadObject(object_path); + if (!metadata.ok()) { + if (!metadata.status().IsNotExist()) { + return metadata.status(); + } + PAIMON_ASSIGN_OR_RAISE(bool is_dir, DirectoryExists(object_path)); + if (is_dir) { + return Status::Invalid(fmt::format("{} is a directory", path)); + } + return metadata.status(); + } + return std::make_unique(client_, read_ahead_limiter_, object_path, + ToUri(object_path), metadata.value().size); +} + +Result> ObjectStoreFileSystem::GetFileStatus( + const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); + if (!object_path.key.empty()) { + Result metadata = client_->HeadObject(object_path); + if (metadata.ok()) { + return std::make_unique( + ToUri(object_path), metadata.value().size, metadata.value().modification_time, + false); + } + if (!metadata.status().IsNotExist()) { + return metadata.status(); + } + } + PAIMON_ASSIGN_OR_RAISE(bool exists, DirectoryExists(object_path)); + if (!exists) { + return Status::NotExist(fmt::format("{} does not exist", path)); + } + return std::make_unique(ToUri(object_path, true), 0, 0, true); +} + +Status ObjectStoreFileSystem::ListDirectory( + const ObjectStorePath& path, std::vector>* basic_statuses, + std::vector>* statuses) const { + if (basic_statuses == nullptr && statuses == nullptr) { + return Status::Invalid("a destination status list is required"); + } + ObjectStorePath directory{path.bucket, NormalizeDirectoryPrefix(path.key)}; + std::string token; + do { + PAIMON_ASSIGN_OR_RAISE(ListObjectsResult result, client_->ListObjects(directory, token, 0)); + if (result.is_truncated && result.continuation_token.empty()) { + return Status::IOError( + fmt::format("truncated listing for {} did not include a continuation token", + ToUri(directory, true))); + } + for (const auto& object : result.objects) { + if (object.key == directory.key) { + continue; + } + ObjectStorePath child{path.bucket, object.key}; + if (basic_statuses) { + basic_statuses->push_back( + std::make_unique(ToUri(child), false)); + } else { + statuses->push_back(std::make_unique( + ToUri(child), object.size, object.modification_time, false)); + } + } + for (const auto& prefix : result.common_prefixes) { + ObjectStorePath child{path.bucket, prefix}; + if (basic_statuses) { + basic_statuses->push_back( + std::make_unique(ToUri(child, true), true)); + } else { + statuses->push_back( + std::make_unique(ToUri(child, true), 0, 0, true)); + } + } + token = result.continuation_token; + if (!result.is_truncated) { + break; + } + } while (true); + return Status::OK(); +} + +Status ObjectStoreFileSystem::ListDir( + const std::string& directory, + std::vector>* file_status_list) const { + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath path, ParsePath(directory)); + if (!path.key.empty() && path.key.back() != '/') { + Result metadata = client_->HeadObject(path); + if (metadata.ok()) { + return Status::Invalid(fmt::format("file {} exists and is not a directory", directory)); + } + if (!metadata.status().IsNotExist()) { + return metadata.status(); + } + } + return ListDirectory(path, file_status_list, nullptr); +} + +Status ObjectStoreFileSystem::ListFileStatus( + const std::string& path, std::vector>* file_status_list) const { + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); + if (!object_path.key.empty() && object_path.key.back() != '/') { + Result metadata = client_->HeadObject(object_path); + if (metadata.ok()) { + file_status_list->push_back( + std::make_unique(ToUri(object_path), metadata.value().size, + metadata.value().modification_time, false)); + return Status::OK(); + } + if (!metadata.status().IsNotExist()) { + return metadata.status(); + } + } + return ListDirectory(object_path, nullptr, file_status_list); +} + +Result ObjectStoreFileSystem::Exists(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path)); + if (!object_path.key.empty()) { + Result metadata = client_->HeadObject(object_path); + if (metadata.ok()) { + return true; + } + if (!metadata.status().IsNotExist()) { + return metadata.status(); + } + } + return DirectoryExists(object_path); +} + +Status ObjectStoreFileSystem::ReadOnlyStatus() const { + return Status::NotImplemented(fmt::format("{} object store file system is read-only", scheme_)); +} + +Result> ObjectStoreFileSystem::Create(const std::string&, + bool) const { + return ReadOnlyStatus(); +} + +Status ObjectStoreFileSystem::Mkdirs(const std::string&) const { + return ReadOnlyStatus(); +} + +Status ObjectStoreFileSystem::Rename(const std::string&, const std::string&) const { + return ReadOnlyStatus(); +} + +Status ObjectStoreFileSystem::Delete(const std::string&, bool) const { + return ReadOnlyStatus(); +} + +Status ObjectStoreFileSystem::WriteFile(const std::string&, const std::string&, bool) { + return ReadOnlyStatus(); +} + +Status ObjectStoreFileSystem::AtomicStore(const std::string&, const std::string&) { + return ReadOnlyStatus(); +} + +} // namespace paimon diff --git a/src/paimon/common/fs/object_store_file_system.h b/src/paimon/common/fs/object_store_file_system.h new file mode 100644 index 00000000..9ef30528 --- /dev/null +++ b/src/paimon/common/fs/object_store_file_system.h @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/fs/file_system.h" +#include "paimon/visibility.h" + +namespace paimon { + +struct ObjectStorePath { + std::string bucket; + std::string key; +}; + +struct ObjectMetadata { + std::string key; + int64_t size = 0; + int64_t modification_time = 0; +}; + +struct ListObjectsResult { + std::vector objects; + std::vector common_prefixes; + std::string continuation_token; + bool is_truncated = false; +}; + +class PAIMON_EXPORT ObjectStoreClient { + public: + virtual ~ObjectStoreClient() = default; + + virtual Result HeadObject(const ObjectStorePath& path) const = 0; + virtual Result ListObjects(const ObjectStorePath& path, + const std::string& continuation_token, + int32_t max_keys) const = 0; + virtual Result GetObjectRange(const ObjectStorePath& path, int64_t offset, + int64_t size, char* buffer) const = 0; + virtual void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size, + char* buffer, + std::function&& callback) const = 0; +}; + +class ReadAheadMemoryLimiter { + public: + explicit ReadAheadMemoryLimiter(int64_t limit); + + int64_t ReserveUpTo(int64_t min_size, int64_t max_size); + void Release(int64_t size); + + private: + int64_t limit_; + int64_t used_ = 0; + std::mutex mutex_; +}; + +class PAIMON_EXPORT ObjectStoreFileSystem : public FileSystem { + public: + ObjectStoreFileSystem(std::string scheme, std::shared_ptr client); + ObjectStoreFileSystem(std::string scheme, std::shared_ptr client, + int64_t read_ahead_memory_limit); + ~ObjectStoreFileSystem() override = default; + + Result> Open(const std::string& path) const override; + Result> GetFileStatus(const std::string& path) const override; + Status ListDir(const std::string& directory, + std::vector>* file_status_list) const override; + Status ListFileStatus( + const std::string& path, + std::vector>* file_status_list) const override; + Result Exists(const std::string& path) const override; + + Result> Create(const std::string& path, + bool overwrite) const override; + Status Mkdirs(const std::string& path) const override; + Status Rename(const std::string& src, const std::string& dst) const override; + Status Delete(const std::string& path, bool recursive = true) const override; + Status WriteFile(const std::string& path, const std::string& content, bool overwrite) override; + Status AtomicStore(const std::string& path, const std::string& content) override; + + protected: + Result ParsePath(const std::string& path) const; + std::string ToUri(const ObjectStorePath& path, bool is_dir = false) const; + + private: + Result DirectoryExists(const ObjectStorePath& path) const; + Status ListDirectory(const ObjectStorePath& path, + std::vector>* basic_statuses, + std::vector>* statuses) const; + Status ReadOnlyStatus() const; + + std::string scheme_; + std::shared_ptr client_; + std::shared_ptr read_ahead_limiter_; +}; + +} // namespace paimon diff --git a/src/paimon/common/fs/object_store_file_system_test.cpp b/src/paimon/common/fs/object_store_file_system_test.cpp new file mode 100644 index 00000000..1b34917a --- /dev/null +++ b/src/paimon/common/fs/object_store_file_system_test.cpp @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/fs/object_store_file_system.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +using Range = std::pair; + +class MockObjectStoreClient : public ObjectStoreClient { + public: + Result HeadObject(const ObjectStorePath& path) const override { + ++head_calls_; + if (!head_error_.ok()) { + return head_error_; + } + auto iter = objects_.find(path.key); + if (iter == objects_.end()) { + return Status::NotExist("not found"); + } + return ObjectMetadata{path.key, static_cast(iter->second.size()), 1}; + } + + Result ListObjects(const ObjectStorePath& path, const std::string& token, + int32_t) const override { + ++list_calls_; + if (!list_error_.ok()) { + return list_error_; + } + if (!pages_.empty()) { + return pages_.at(token); + } + ListObjectsResult result; + for (const auto& [key, value] : objects_) { + if (key.rfind(path.key, 0) == 0) { + result.objects.push_back( + ObjectMetadata{key, static_cast(value.size()), 1}); + } + } + return result; + } + + Result GetObjectRange(const ObjectStorePath& path, int64_t offset, int64_t size, + char* buffer) const override { + ranges_.emplace_back(offset, size); + const std::string& value = objects_.at(path.key); + int64_t available = std::min(size, static_cast(value.size()) - offset); + if (short_read_) { + --available; + } + std::memcpy(buffer, value.data() + offset, available); + return available; + } + + void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size, + char* buffer, std::function&& callback) const override { + Result result = GetObjectRange(path, offset, size, buffer); + callback(result.ok() && result.value() == size ? Status::OK() + : Status::IOError("short read")); + } + + std::map objects_; + std::map pages_; + Status head_error_ = Status::OK(); + Status list_error_ = Status::OK(); + mutable int32_t list_calls_ = 0; + mutable int32_t head_calls_ = 0; + mutable std::vector ranges_; + bool short_read_ = false; +}; + +TEST(ObjectStoreFileSystemTest, TestObjectWinsOverPrefix) { + auto client = std::make_shared(); + client->objects_["foo"] = "file"; + client->objects_["foo/bar"] = "child"; + ObjectStoreFileSystem fs("s3", client); + ASSERT_OK_AND_ASSIGN(auto status, fs.GetFileStatus("s3://bucket/foo")); + ASSERT_FALSE(status->IsDir()); + std::vector> statuses; + ASSERT_OK(fs.ListFileStatus("s3://bucket/foo", &statuses)); + ASSERT_EQ(statuses.size(), 1); + ASSERT_FALSE(statuses[0]->IsDir()); +} + +TEST(ObjectStoreFileSystemTest, TestHeadErrorIsNotMasked) { + auto client = std::make_shared(); + client->head_error_ = Status::IOError("access denied"); + client->objects_["foo/bar"] = "child"; + ObjectStoreFileSystem fs("s3", client); + auto status = fs.GetFileStatus("s3://bucket/foo"); + ASSERT_TRUE(status.status().IsIOError()); + ASSERT_EQ(client->list_calls_, 0); +} + +TEST(ObjectStoreFileSystemTest, TestPaginationAndDirectoryMarker) { + auto client = std::make_shared(); + ListObjectsResult first; + first.objects.push_back({"dir/", 0, 0}); + first.objects.push_back({"dir/a", 1, 1}); + first.is_truncated = true; + first.continuation_token = "next"; + ListObjectsResult second; + second.common_prefixes.push_back("dir/sub/"); + client->pages_[""] = first; + client->pages_["next"] = second; + ObjectStoreFileSystem fs("s3", client); + std::vector> statuses; + ASSERT_OK(fs.ListFileStatus("s3://bucket/dir/", &statuses)); + ASSERT_EQ(statuses.size(), 2); + ASSERT_EQ(statuses[0]->GetPath(), "s3://bucket/dir/a"); + ASSERT_TRUE(statuses[1]->IsDir()); +} + +TEST(ObjectStoreFileSystemTest, TestTruncatedPageRequiresContinuationToken) { + auto client = std::make_shared(); + ListObjectsResult page; + page.objects.push_back({"dir/a", 1, 1}); + page.is_truncated = true; + client->pages_[""] = page; + ObjectStoreFileSystem fs("s3", client); + std::vector> statuses; + ASSERT_TRUE(fs.ListFileStatus("s3://bucket/dir/", &statuses).IsIOError()); + ASSERT_TRUE(statuses.empty()); +} + +TEST(ObjectStoreFileSystemTest, TestOpenBucketRootIsDirectory) { + auto client = std::make_shared(); + ObjectStoreFileSystem fs("s3", client); + ASSERT_TRUE(fs.Open("s3://bucket/").status().IsInvalid()); + ASSERT_EQ(client->head_calls_, 0); + ASSERT_EQ(client->list_calls_, 0); +} + +TEST(ObjectStoreFileSystemTest, TestPathWithLeadingSlashes) { + auto client = std::make_shared(); + client->objects_["file"] = "data"; + ObjectStoreFileSystem fs("s3", client); + ASSERT_OK_AND_ASSIGN(auto status, fs.GetFileStatus("s3://bucket///file")); + ASSERT_EQ(status->GetPath(), "s3://bucket/file"); +} + +TEST(ObjectStoreInputStreamTest, TestBoundsCloseAndSeekOverflow) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(128 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client, 64 * 1024); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + char data[4]; + bool called = false; + stream->ReadAsync(data, 4, 128 * 1024 - 2, [&called](Status status) { + called = true; + ASSERT_TRUE(status.IsInvalid()); + }); + ASSERT_TRUE(called); + ASSERT_TRUE(stream->Seek(std::numeric_limits::max(), FS_SEEK_END).IsInvalid()); + ASSERT_OK(stream->Close()); + called = false; + stream->ReadAsync(data, 1, 0, [&called, &stream](Status status) { + called = true; + ASSERT_TRUE(status.IsIOError()); + ASSERT_TRUE(stream->GetPos().status().IsIOError()); + }); + ASSERT_TRUE(called); +} + +TEST(ObjectStoreInputStreamTest, TestShortReadFails) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(128 * 1024, 'x'); + client->short_read_ = true; + ObjectStoreFileSystem fs("s3", client); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + char data[4]; + ASSERT_TRUE(stream->Read(data, sizeof(data)).status().IsIOError()); +} + +TEST(ObjectStoreInputStreamTest, TestSequentialReadAheadGrows) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(1024 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + std::array buffer{}; + ASSERT_OK(stream->Read(buffer.data(), buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024)); + ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET)); + ASSERT_OK(stream->Read(buffer.data(), buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(64 * 1024, 128 * 1024)); +} + +TEST(ObjectStoreInputStreamTest, TestConsumedBufferReleasesBudget) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(1024 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client, 64 * 1024); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + std::array buffer{}; + for (int32_t i = 0; i < 17; ++i) { + ASSERT_OK(stream->Read(buffer.data(), buffer.size())); + } + ASSERT_EQ(client->ranges_.size(), 2); + ASSERT_EQ(client->ranges_.back(), Range(64 * 1024, 64 * 1024)); +} + +TEST(ObjectStoreInputStreamTest, TestBackwardSeekReleasesBudget) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(1024 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client, 64 * 1024); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + std::array buffer{}; + ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET)); + ASSERT_OK(stream->Read(buffer.data(), buffer.size())); + ASSERT_OK(stream->Seek(0, FS_SEEK_SET)); + ASSERT_OK(stream->Read(buffer.data(), buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024)); +} + +TEST(ObjectStoreInputStreamTest, TestLargeDirectReadReleasesBudget) { + auto client = std::make_shared(); + client->objects_["file"] = std::string(16 * 1024 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client, 64 * 1024); + ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file")); + std::array small_buffer{}; + ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET)); + ASSERT_OK(stream->Read(small_buffer.data(), small_buffer.size())); + std::vector large_buffer(8 * 1024 * 1024); + ASSERT_OK(stream->Seek(1024 * 1024, FS_SEEK_SET)); + ASSERT_OK(stream->Read(large_buffer.data(), large_buffer.size())); + ASSERT_OK(stream->Seek(0, FS_SEEK_SET)); + ASSERT_OK(stream->Read(small_buffer.data(), small_buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024)); +} + +TEST(ObjectStoreInputStreamTest, TestCompetingStreamsShareBudget) { + auto client = std::make_shared(); + client->objects_["first"] = std::string(1024 * 1024, 'x'); + client->objects_["second"] = std::string(1024 * 1024, 'x'); + ObjectStoreFileSystem fs("s3", client, 64 * 1024); + ASSERT_OK_AND_ASSIGN(auto first, fs.Open("s3://bucket/first")); + ASSERT_OK_AND_ASSIGN(auto second, fs.Open("s3://bucket/second")); + std::array buffer{}; + ASSERT_OK(first->Read(buffer.data(), buffer.size())); + ASSERT_OK(second->Read(buffer.data(), buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(0, 4096)); + ASSERT_OK(first->Close()); + ASSERT_OK(second->Seek(100000, FS_SEEK_SET)); + ASSERT_OK(second->Read(buffer.data(), buffer.size())); + ASSERT_EQ(client->ranges_.back(), Range(100000, 64 * 1024)); +} + +} // namespace +} // namespace paimon::test diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index 7805e3a6..0bfad32e 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -40,7 +40,6 @@ add_paimon_lib(paimon_global_index STATIC_LINK_LIBS arrow fmt - dl Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp index e0a8f54f..2975c90c 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp @@ -22,6 +22,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> BitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.h b/src/paimon/common/global_index/bitmap/bitmap_global_index.h index 60b11fcb..7045dcb8 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.h +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -32,6 +33,8 @@ class BitmapGlobalIndex : public GlobalIndexer { public: explicit BitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp index 3e222f6e..f19c294c 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp @@ -74,7 +74,6 @@ class BitmapGlobalIndexTest : public ::testing::Test { auto wrapper = std::dynamic_pointer_cast(global_writer); EXPECT_TRUE(wrapper); - wrapper->max_write_size_ = 128; ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index e5e95e92..0adde6c0 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -54,8 +54,7 @@ class BTreeCompatibilityTest : public ::testing::Test { EXPECT_OK_AND_ASSIGN(auto input, fs_->Open(path)); EXPECT_OK_AND_ASSIGN(auto length, input->Length()); std::string buffer(static_cast(length), '\0'); - EXPECT_OK_AND_ASSIGN([[maybe_unused]] auto bytes_read, - input->Read(buffer.data(), static_cast(length))); + EXPECT_OK_AND_ASSIGN([[maybe_unused]] auto bytes_read, input->Read(buffer.data(), length)); return buffer; } @@ -117,7 +116,7 @@ class BTreeCompatibilityTest : public ::testing::Test { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(bin_path)); - auto file_size = static_cast(file_status->GetLen()); + auto file_size = file_status->GetLen(); GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes); std::vector metas = {io_meta}; diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp index 07537949..cedfa4a7 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp @@ -49,11 +49,7 @@ Result> BTreeFileMetaSelector::VisitEqual(const L if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice)); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice)); - return cmp_min >= 0 && cmp_max <= 0; + return Overlaps(meta, literal_slice, literal_slice); }); } @@ -70,8 +66,7 @@ Result> BTreeFileMetaSelector::VisitLessThan( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice)); return cmp < 0; }); } @@ -84,8 +79,7 @@ Result> BTreeFileMetaSelector::VisitLessOrEqual( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice)); return cmp <= 0; }); } @@ -98,8 +92,7 @@ Result> BTreeFileMetaSelector::VisitGreaterThan( if (meta.OnlyNulls()) { return false; } - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice)); return cmp > 0; }); } @@ -112,8 +105,7 @@ Result> BTreeFileMetaSelector::VisitGreaterOrEqua if (meta.OnlyNulls()) { return false; } - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice)); return cmp >= 0; }); } @@ -130,12 +122,9 @@ Result> BTreeFileMetaSelector::VisitIn( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); for (const auto& literal_slice : literal_slices) { - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice)); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice)); - if (cmp_min >= 0 && cmp_max <= 0) { + PAIMON_ASSIGN_OR_RAISE(bool overlaps, Overlaps(meta, literal_slice, literal_slice)); + if (overlaps) { return true; } } @@ -179,6 +168,39 @@ Result> BTreeFileMetaSelector::Filter( return result; } +Result BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, + const MemorySlice& to) const { + if (meta.FirstKey()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(to, WrapKeySlice(meta.FirstKey()))); + if (cmp < 0) { + return false; + } + } + if (meta.LastKey()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(from, WrapKeySlice(meta.LastKey()))); + if (cmp > 0) { + return false; + } + } + return true; +} + +Result BTreeFileMetaSelector::CompareFirstKey(const BTreeIndexMeta& meta, + const MemorySlice& literal) const { + if (!meta.FirstKey()) { + return -1; + } + return comparator_(WrapKeySlice(meta.FirstKey()), literal); +} + +Result BTreeFileMetaSelector::CompareLastKey(const BTreeIndexMeta& meta, + const MemorySlice& literal) const { + if (!meta.LastKey()) { + return 1; + } + return comparator_(WrapKeySlice(meta.LastKey()), literal); +} + MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr& key) { return MemorySlice::Wrap(MemorySegment::WrapView(key->data(), key->size())); } diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h b/src/paimon/common/global_index/btree/btree_file_meta_selector.h index 405efc90..b59fc7ed 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.h @@ -59,6 +59,13 @@ class BTreeFileMetaSelector : public FunctionVisitor> Filter(const MetaPredicate& predicate) const; + Result Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, + const MemorySlice& to) const; + + Result CompareFirstKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + + Result CompareLastKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + Result SerializeLiteral(const Literal& literal) const; /// Create a non-owning MemorySlice view over the raw bytes of a key, diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index 4e532fe1..ce77207b 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -228,4 +228,43 @@ TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { ASSERT_EQ(names.count("file6"), 1u); } +TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr key_type = arrow::utf8(); + auto serialize = [&](const char* value, size_t size) { + Literal literal(FieldType::STRING, value, size); + EXPECT_OK_AND_ASSIGN(std::shared_ptr result, + KeySerializer::SerializeKey(literal, key_type, pool.get())); + return result; + }; + + auto empty_meta = + std::make_shared(serialize("", 0), serialize("www.example.com", 15), false); + auto normal_meta = + std::make_shared(serialize("aaa.com", 7), serialize("zzz.com", 7), false); + auto null_meta = std::make_shared(nullptr, nullptr, true); + std::vector files = { + GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_normal", 1, normal_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), + }; + + BTreeFileMetaSelector selector(files, key_type, pool); + + ASSERT_OK_AND_ASSIGN(std::vector result, + selector.VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); + CheckResult(result, {"file_empty", "file_normal"}); + + ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); + CheckResult(result, {"file_empty", "file_normal"}); + + ASSERT_OK_AND_ASSIGN( + result, selector.VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); + CheckResult(result, {"file_normal"}); + + ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(FieldType::STRING, "", 0), + Literal(FieldType::STRING, "zzz.com", 7)})); + CheckResult(result, {"file_empty", "file_normal"}); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index b05a0a8c..e5aeaac2 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -23,6 +23,7 @@ #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" #include "paimon/common/global_index/btree/btree_global_indexer.h" +#include "paimon/common/global_index/btree/btree_index_meta.h" #include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/utils/scope_guard.h" @@ -55,7 +56,7 @@ class FakeGlobalIndexFileWriter : public GlobalIndexFileWriter { Result GetFileSize(const std::string& file_name) const override { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(base_path_ + "/" + file_name)); - return static_cast(file_status->GetLen()); + return file_status->GetLen(); } std::string ToPath(const std::string& file_name) const override { @@ -460,6 +461,50 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadStringData) { } } +TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { + auto file_writer = std::make_shared(fs_, base_path_); + auto field = arrow::field("str_field", arrow::utf8()); + auto c_schema = CreateArrowSchema(field); + + std::map options = {{BtreeDefs::kBtreeIndexBlockSize, "128"}, + {BtreeDefs::kBtreeIndexCompression, GetParam()}}; + ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); + ASSERT_OK_AND_ASSIGN(auto writer, + indexer->CreateWriter("str_field", c_schema.get(), file_writer, pool_)); + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), R"([ + [null], + [""], + ["abc"] + ])") + .ValueOrDie(); + + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector row_ids(array->length()); + std::iota(row_ids.begin(), row_ids.end(), 0); + ASSERT_OK(writer->AddBatch(&c_array, std::move(row_ids))); + ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); + ASSERT_EQ(metas.size(), 1); + + std::shared_ptr meta = + BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get()); + ASSERT_TRUE(meta->FirstKey()); + ASSERT_EQ(meta->FirstKey()->size(), 0); + ASSERT_TRUE(meta->LastKey()); + ASSERT_EQ(std::string(meta->LastKey()->data(), meta->LastKey()->size()), "abc"); + ASSERT_TRUE(meta->HasNulls()); + ASSERT_FALSE(meta->OnlyNulls()); + + auto file_reader = std::make_shared(fs_, base_path_); + c_schema = CreateArrowSchema(field); + ASSERT_OK_AND_ASSIGN(auto reader, + indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + + Literal empty(FieldType::STRING, "", 0); + ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(empty)); + CheckResult(result, {1}); +} + TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBigIntData) { auto file_writer = std::make_shared(fs_, base_path_); auto field = arrow::field("bigint_field", arrow::int64()); diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index fd410089..29bffe9f 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -57,6 +57,10 @@ Result> BTreeGlobalIndexer::Create( return std::unique_ptr(new BTreeGlobalIndexer(cache_manager, options)); } +Result>> BTreeGlobalIndexer::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BTreeGlobalIndexer::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index d1f5e926..5568adba 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -56,6 +56,8 @@ class BTreeGlobalIndexer : public GlobalIndexer { static Result> Create( const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp b/src/paimon/common/global_index/btree/btree_index_meta.cpp index 94d7b44c..9ba88e53 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta.cpp @@ -22,48 +22,71 @@ #include "paimon/common/memory/memory_slice_output.h" namespace paimon { +namespace { + +std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, MemoryPool* pool) { + if (key_length == 0) { + return std::make_shared(0, pool); + } + return input->ReadSliceView(key_length).CopyBytes(pool); +} + +} // namespace std::shared_ptr BTreeIndexMeta::Deserialize(const std::shared_ptr& meta, paimon::MemoryPool* pool) { - auto slice = MemorySlice::Wrap(meta); - auto input = slice.ToInput(); - auto first_key_len = input.ReadInt(); - std::shared_ptr first_key; - if (first_key_len) { - first_key = input.ReadSliceView(first_key_len).CopyBytes(pool); - } - auto last_key_len = input.ReadInt(); - std::shared_ptr last_key; - if (last_key_len) { - last_key = input.ReadSliceView(last_key_len).CopyBytes(pool); + MemorySlice slice = MemorySlice::Wrap(meta); + MemorySliceInput input = slice.ToInput(); + int32_t first_key_len = input.ReadInt(); + std::shared_ptr first_key = ReadKey(&input, first_key_len, pool); + int32_t last_key_len = input.ReadInt(); + std::shared_ptr last_key = ReadKey(&input, last_key_len, pool); + bool has_nulls = input.ReadByte() == static_cast(1); + + if (input.Available() >= 2) { + int8_t format_version = input.ReadByte(); + if (format_version == kFormatVersionWithNullFlags) { + int8_t null_key_flags = input.ReadByte(); + if ((null_key_flags & kFirstKeyIsNull) != 0) { + first_key.reset(); + } + if ((null_key_flags & kLastKeyIsNull) != 0) { + last_key.reset(); + } + } + } else if (first_key_len == 0 && last_key_len == 0 && has_nulls) { + // Legacy metadata used zero length for null keys. Both empty boundaries plus a null bitmap + // identify an all-null file; a single empty boundary remains a valid serialized key. + first_key.reset(); + last_key.reset(); } - auto has_nulls = input.ReadByte() == static_cast(1); return std::make_shared(first_key, last_key, has_nulls); } std::shared_ptr BTreeIndexMeta::Serialize(paimon::MemoryPool* pool) const { - // Calculate total size: first_key_len(4) + first_key + last_key_len(4) + last_key + - // has_nulls(1) int32_t first_key_size = first_key_ ? first_key_->size() : 0; int32_t last_key_size = last_key_ ? last_key_->size() : 0; int32_t total_size = Size(); - MemorySliceOutput output(total_size, pool); + int8_t null_key_flags = 0; - // Write first_key_len and first_key output.WriteValue(first_key_size); if (first_key_) { output.WriteBytes(first_key_); + } else { + null_key_flags |= kFirstKeyIsNull; } - // Write last_key_len and last_key output.WriteValue(last_key_size); if (last_key_) { output.WriteBytes(last_key_); + } else { + null_key_flags |= kLastKeyIsNull; } - // Write has_nulls output.WriteValue(static_cast(has_nulls_ ? 1 : 0)); + output.WriteValue(kFormatVersionWithNullFlags); + output.WriteValue(null_key_flags); return output.ToSlice().GetOrCreateHeapMemory(pool); } diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h b/src/paimon/common/global_index/btree/btree_index_meta.h index 05f08f91..85df5ff3 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.h +++ b/src/paimon/common/global_index/btree/btree_index_meta.h @@ -19,14 +19,16 @@ #pragma once +#include #include #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" namespace paimon { -/// Index Meta of each BTree index file. The first key and last key of this meta could be null if -/// the entire btree index file only contains nulls. +/// Index metadata for each BTree index file. +/// +/// Empty serialized keys are valid, so null boundary keys are encoded separately with flags. class BTreeIndexMeta { public: static std::shared_ptr Deserialize(const std::shared_ptr& meta, @@ -56,11 +58,15 @@ class BTreeIndexMeta { private: int32_t Size() const { - // 9 bytes => first_key_len(4 byte) + last_key_len(4 byte) + has_null(1 byte) - return (first_key_ ? first_key_->size() : 0) + (last_key_ ? last_key_->size() : 0) + 9; + // 11 bytes => key lengths (8) + has_nulls (1) + format version (1) + null flags (1). + return (first_key_ ? first_key_->size() : 0) + (last_key_ ? last_key_->size() : 0) + 11; } private: + static constexpr int8_t kFormatVersionWithNullFlags = 1; + static constexpr int8_t kFirstKeyIsNull = 1; + static constexpr int8_t kLastKeyIsNull = 1 << 1; + std::shared_ptr first_key_; std::shared_ptr last_key_; bool has_nulls_; diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp index c29ec6be..70c4dfcb 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp @@ -20,6 +20,7 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" #include "gtest/gtest.h" +#include "paimon/common/memory/memory_slice_output.h" #include "paimon/memory/memory_pool.h" namespace paimon::test { @@ -29,6 +30,24 @@ class BTreeIndexMetaTest : public ::testing::Test { pool_ = GetDefaultPool(); } + std::shared_ptr LegacyMetaBytes(const std::shared_ptr& first_key, + const std::shared_ptr& last_key, + bool has_nulls) const { + int32_t first_key_size = first_key ? first_key->size() : 0; + int32_t last_key_size = last_key ? last_key->size() : 0; + MemorySliceOutput output(first_key_size + last_key_size + 9, pool_.get()); + output.WriteValue(first_key_size); + if (first_key) { + output.WriteBytes(first_key); + } + output.WriteValue(last_key_size); + if (last_key) { + output.WriteBytes(last_key); + } + output.WriteValue(static_cast(has_nulls ? 1 : 0)); + return output.ToSlice().CopyBytes(pool_.get()); + } + std::shared_ptr pool_; }; @@ -61,29 +80,93 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_TRUE(deserialized->HasNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyKeys) { - // Create a BTreeIndexMeta with empty keys (OnlyNulls case) +TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { + auto empty_key = std::make_shared(0, pool_.get()); + auto last_key = std::make_shared("last_key_data", pool_.get()); + auto meta = std::make_shared(empty_key, last_key, false); + + auto serialized = meta->Serialize(pool_.get()); + ASSERT_EQ(serialized->size(), 11 + last_key->size()); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(std::string(deserialized->LastKey()->data(), deserialized->LastKey()->size()), + "last_key_data"); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto meta = std::make_shared(empty_key, empty_key, true); + + auto serialized = meta->Serialize(pool_.get()); + ASSERT_EQ(serialized->size(), 11); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(deserialized->LastKey()->size(), 0); + ASSERT_TRUE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { auto meta = std::make_shared(nullptr, nullptr, true); - // Serialize auto serialized = meta->Serialize(pool_.get()); - ASSERT_TRUE(serialized); + ASSERT_EQ(serialized->size(), 11); - // Deserialize auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); ASSERT_TRUE(deserialized); - - // Verify keys are null ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); - - // Verify has_nulls ASSERT_TRUE(deserialized->HasNulls()); + ASSERT_TRUE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, empty_key, true); - // Verify OnlyNulls + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_FALSE(deserialized->FirstKey()); + ASSERT_FALSE(deserialized->LastKey()); + ASSERT_TRUE(deserialized->HasNulls()); ASSERT_TRUE(deserialized->OnlyNulls()); } +TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { + auto empty_key = std::make_shared(0, pool_.get()); + auto last_key = std::make_shared("last_key_data", pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, last_key, false); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(std::string(deserialized->LastKey()->data(), deserialized->LastKey()->size()), + "last_key_data"); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, empty_key, false); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(deserialized->LastKey()->size(), 0); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) { // Case 1: Has nulls with keys auto meta1 = diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index 655efa91..6a45aee3 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -55,7 +55,7 @@ class FakeLazyFileWriter : public GlobalIndexFileWriter { Result GetFileSize(const std::string& file_name) const override { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(base_path_ + "/" + file_name)); - return static_cast(file_status->GetLen()); + return file_status->GetLen(); } std::string ToPath(const std::string& file_name) const override { diff --git a/src/paimon/common/global_index/global_index_utils_test.cpp b/src/paimon/common/global_index/global_index_utils_test.cpp index c31aebd0..7354bced 100644 --- a/src/paimon/common/global_index/global_index_utils_test.cpp +++ b/src/paimon/common/global_index/global_index_utils_test.cpp @@ -34,7 +34,9 @@ class GlobalIndexUtilsTest : public ::testing::Test { /// Helper to create a valid ArrowArray with the given number of int32 elements. static ArrowArray CreateInt32Array(const std::vector& values) { arrow::Int32Builder builder; - EXPECT_TRUE(builder.AppendValues(values).ok()); + if (!values.empty()) { + EXPECT_TRUE(builder.AppendValues(values).ok()); + } std::shared_ptr array; EXPECT_TRUE(builder.Finish(&array).ok()); ArrowArray c_array; diff --git a/src/paimon/common/global_index/global_indexer_factory.cpp b/src/paimon/common/global_index/global_indexer_factory.cpp index 11dbc981..f40a42d1 100644 --- a/src/paimon/common/global_index/global_indexer_factory.cpp +++ b/src/paimon/common/global_index/global_indexer_factory.cpp @@ -31,6 +31,17 @@ const char GlobalIndexerFactory::GLOBAL_INDEX_IDENTIFIER_SUFFIX[] = "-global"; Result> GlobalIndexerFactory::Get( const std::string& identifier, const std::map& options) { + // Java now uses a dedicated bitmap global index format instead of the previously shared + // wrapped file index format. Keep the legacy implementation registered for a future migration, + // but do not expose it as a compatible global index. + static constexpr const char* kEnableLegacyBitmapForTesting = + "bitmap-global-index.legacy-format.enabled-for-testing"; + auto enable_legacy_bitmap = options.find(kEnableLegacyBitmapForTesting); + if (identifier == "bitmap" && + (enable_legacy_bitmap == options.end() || enable_legacy_bitmap->second != "true")) { + return std::unique_ptr(); + } + // Compatibility: "lumina-vector-ann" was the old identifier for lumina global index. std::string final_identifier = (identifier == "lumina-vector-ann" ? "lumina" : identifier); std::string global_index_identifier = final_identifier + GLOBAL_INDEX_IDENTIFIER_SUFFIX; diff --git a/src/paimon/common/global_index/global_indexer_factory_test.cpp b/src/paimon/common/global_index/global_indexer_factory_test.cpp index 77e819ce..c03fec58 100644 --- a/src/paimon/common/global_index/global_indexer_factory_test.cpp +++ b/src/paimon/common/global_index/global_indexer_factory_test.cpp @@ -26,13 +26,19 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(GlobalIndexerFactoryTest, TestSimple) { +TEST(GlobalIndexerFactoryTest, TestBitmapUnsupported) { std::map options; ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, GlobalIndexerFactory::Get("bitmap", options)); + ASSERT_FALSE(indexer); +} - auto bitmap_global_index = dynamic_cast(indexer.get()); - ASSERT_TRUE(bitmap_global_index); +TEST(GlobalIndexerFactoryTest, TestLegacyBitmapEnabledForTesting) { + std::map options = { + {"bitmap-global-index.legacy-format.enabled-for-testing", "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, + GlobalIndexerFactory::Get("bitmap", options)); + ASSERT_TRUE(dynamic_cast(indexer.get())); } TEST(GlobalIndexerFactoryTest, TestNonExist) { diff --git a/src/paimon/common/global_index/offset_global_index_reader_test.cpp b/src/paimon/common/global_index/offset_global_index_reader_test.cpp index e090045a..0815f3f1 100644 --- a/src/paimon/common/global_index/offset_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/offset_global_index_reader_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/roaring_bitmap64.h" @@ -115,9 +116,14 @@ class FakeGlobalIndexReader : public GlobalIndexReader { Result> VisitFullTextSearch( const std::shared_ptr& full_text_search) override { + captured_fts = full_text_search; return MakeResult(default_result_); } + // Captures the (possibly pre_filter-rewritten) FullTextSearch the offset + // reader forwarded, so tests can assert field propagation. + std::shared_ptr captured_fts; + bool IsThreadSafe() const override { return true; } @@ -334,6 +340,37 @@ TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchWithOffset) { CheckResult(result, {10, 13, 15}); } +TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchPreservesScoreFlags) { + // Regression (review finding #2): rewriting the pre_filter global->local ids + // in the offset reader must NOT drop with_score / min_score. Before the fix, + // FullTextSearch::ReplacePreFilter rebuilt via the 5-arg ctor and silently + // reset both back to their defaults, turning a scored / min_score query + // unscored as soon as it crossed any offset shard. + auto fake_reader = std::make_shared(); + fake_reader->SetDefaultResult({0, 3, 5}); + auto offset_reader = std::make_shared(fake_reader, 10); + + // pre_filter must be set so the offset reader takes the rewrite path. + auto fts = std::make_shared( + "f0", /*limit=*/7, "q", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({10l, 13l, 15l})); + fts->with_score = true; + fts->min_score = 1.5f; + + ASSERT_OK_AND_ASSIGN(auto result, offset_reader->VisitFullTextSearch(fts)); + CheckResult(result, {10, 13, 15}); + + ASSERT_TRUE(fake_reader->captured_fts); + ASSERT_TRUE(fake_reader->captured_fts->with_score) + << "with_score must survive the pre_filter rewrite"; + ASSERT_TRUE(fake_reader->captured_fts->min_score.has_value()) + << "min_score must survive the pre_filter rewrite"; + ASSERT_FLOAT_EQ(fake_reader->captured_fts->min_score.value(), 1.5f); + // limit and the offset-rewritten local pre_filter should still be present. + ASSERT_EQ(fake_reader->captured_fts->limit, std::optional(7)); + ASSERT_TRUE(fake_reader->captured_fts->pre_filter.has_value()); +} + TEST_F(OffsetGlobalIndexReaderTest, TestVisitVectorSearchWithOffset) { auto fake_reader = std::make_shared(); fake_reader->SetVectorSearchResult({0, 2, 5}, {0.9f, 0.7f, 0.3f}); diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp index db3c756e..71f9f4bc 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp @@ -22,6 +22,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> RangeBitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> RangeBitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h index 7b8923e4..3e60c428 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -33,6 +34,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer { explicit RangeBitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp index 0e032177..01a1f842 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp @@ -73,7 +73,6 @@ class RangeBitmapGlobalIndexTest : public ::testing::Test { auto wrapper = std::dynamic_pointer_cast(global_writer); EXPECT_TRUE(wrapper); - wrapper->max_write_size_ = 128; ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); diff --git a/src/paimon/common/global_index/union_global_index_reader.cpp b/src/paimon/common/global_index/union_global_index_reader.cpp index a4de3b21..1b52d0a9 100644 --- a/src/paimon/common/global_index/union_global_index_reader.cpp +++ b/src/paimon/common/global_index/union_global_index_reader.cpp @@ -42,84 +42,84 @@ Result> UnionGlobalIndexReader::VisitIsNull() Result> UnionGlobalIndexReader::VisitEqual( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitEqual(literal); }); } Result> UnionGlobalIndexReader::VisitNotEqual( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitNotEqual(literal); }); } Result> UnionGlobalIndexReader::VisitLessThan( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitLessThan(literal); }); } Result> UnionGlobalIndexReader::VisitLessOrEqual( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitLessOrEqual(literal); }); } Result> UnionGlobalIndexReader::VisitGreaterThan( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitGreaterThan(literal); }); } Result> UnionGlobalIndexReader::VisitGreaterOrEqual( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitGreaterOrEqual(literal); }); } Result> UnionGlobalIndexReader::VisitIn( const std::vector& literals) { - return Union([&literals](const std::shared_ptr& reader) { + return Union([literals](const std::shared_ptr& reader) { return reader->VisitIn(literals); }); } Result> UnionGlobalIndexReader::VisitNotIn( const std::vector& literals) { - return Union([&literals](const std::shared_ptr& reader) { + return Union([literals](const std::shared_ptr& reader) { return reader->VisitNotIn(literals); }); } Result> UnionGlobalIndexReader::VisitStartsWith( const Literal& prefix) { - return Union([&prefix](const std::shared_ptr& reader) { + return Union([prefix](const std::shared_ptr& reader) { return reader->VisitStartsWith(prefix); }); } Result> UnionGlobalIndexReader::VisitEndsWith( const Literal& suffix) { - return Union([&suffix](const std::shared_ptr& reader) { + return Union([suffix](const std::shared_ptr& reader) { return reader->VisitEndsWith(suffix); }); } Result> UnionGlobalIndexReader::VisitContains( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitContains(literal); }); } Result> UnionGlobalIndexReader::VisitLike( const Literal& literal) { - return Union([&literal](const std::shared_ptr& reader) { + return Union([literal](const std::shared_ptr& reader) { return reader->VisitLike(literal); }); } @@ -127,7 +127,7 @@ Result> UnionGlobalIndexReader::VisitLike( Result> UnionGlobalIndexReader::VisitVectorSearch( const std::shared_ptr& vector_search) { auto results = ExecuteAllReaders>>( - [&vector_search](const std::shared_ptr& reader) + [vector_search](const std::shared_ptr& reader) -> Result> { return reader->VisitVectorSearch(vector_search); }); @@ -158,15 +158,13 @@ Result> UnionGlobalIndexReader::VisitVe Result> UnionGlobalIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { - return Union([&full_text_search](const std::shared_ptr& reader) { + return Union([full_text_search](const std::shared_ptr& reader) { return reader->VisitFullTextSearch(full_text_search); }); } Result> UnionGlobalIndexReader::Union(ReaderAction action) { - auto results = ExecuteAllReaders>>( - [&action](const std::shared_ptr& reader) - -> Result> { return action(reader); }); + auto results = ExecuteAllReaders>>(action); std::shared_ptr merged_result = nullptr; for (auto& result_or_status : results) { @@ -210,8 +208,7 @@ std::vector UnionGlobalIndexReader::ExecuteAllReaders( std::vector> futures; futures.reserve(readers_.size()); for (const auto& reader : readers_) { - futures.push_back( - Via(executor_.get(), [&action, reader]() -> R { return action(reader); })); + futures.push_back(Via(executor_.get(), [action, reader]() -> R { return action(reader); })); } return CollectAll(futures); } diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp index 8d55db42..6c3c6858 100644 --- a/src/paimon/common/global_index/union_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -57,6 +59,11 @@ class FakeReader : public GlobalIndexReader { error_message_ = message; } + void SetThrowException(const std::string& message) { + throw_exception_ = true; + exception_message_ = message; + } + /// Sets a scored result returned by VisitVectorSearch. void SetScoredResult(const std::vector& row_ids, const std::vector& scores) { scored_row_ids_ = row_ids; @@ -164,6 +171,9 @@ class FakeReader : public GlobalIndexReader { private: Result> MakeResult() { invocation_count_++; + if (throw_exception_) { + throw std::runtime_error(exception_message_); + } if (return_error_) { return Status::Invalid(error_message_); } @@ -179,7 +189,9 @@ class FakeReader : public GlobalIndexReader { std::vector default_result_; bool return_nullptr_ = false; bool return_error_ = false; + bool throw_exception_ = false; std::string error_message_; + std::string exception_message_; std::vector scored_row_ids_; std::vector scored_scores_; bool has_scored_result_ = false; @@ -187,6 +199,40 @@ class FakeReader : public GlobalIndexReader { std::atomic invocation_count_{0}; }; +// Runs the first task immediately and defers the rest. This makes it possible to verify that +// queued tasks own their action even if collecting an earlier future throws. +class DeferAfterFirstExecutor : public Executor { + public: + void Add(std::function func) override { + if (submission_count_++ == 0) { + func(); + } else { + pending_tasks_.push(std::move(func)); + } + } + + void ShutdownNow() override { + std::queue> empty; + pending_tasks_.swap(empty); + } + + uint32_t GetThreadNum() const override { + return 1; + } + + void RunPendingTasks() { + while (!pending_tasks_.empty()) { + std::function task = std::move(pending_tasks_.front()); + pending_tasks_.pop(); + task(); + } + } + + private: + uint32_t submission_count_ = 0; + std::queue> pending_tasks_; +}; + class UnionGlobalIndexReaderTest : public ::testing::Test { public: static void CheckResult(const std::shared_ptr& result, @@ -344,6 +390,23 @@ TEST_F(UnionGlobalIndexReaderTest, TestErrorPropagationWithExecutor) { ASSERT_NOK_WITH_MSG(union_reader.VisitIsNotNull(), "Unknown error for reader2"); } +TEST_F(UnionGlobalIndexReaderTest, TestDeferredTaskOwnsActionAfterEarlierFutureThrows) { + auto throwing_reader = std::make_shared(); + auto deferred_reader = std::make_shared(); + throwing_reader->SetThrowException("reader exception"); + deferred_reader->SetDefaultResult({2}); + + std::vector> readers = {throwing_reader, deferred_reader}; + auto executor = std::make_shared(); + UnionGlobalIndexReader union_reader(std::move(readers), executor); + ASSERT_THROW( + { [[maybe_unused]] auto result = union_reader.VisitIsNotNull(); }, std::runtime_error); + ASSERT_EQ(deferred_reader->InvocationCount(), 0); + + executor->RunPendingTasks(); + ASSERT_EQ(deferred_reader->InvocationCount(), 1); +} + TEST_F(UnionGlobalIndexReaderTest, TestVisitEqualUnion) { auto reader1 = std::make_shared(); auto reader2 = std::make_shared(); diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index 90a7b84d..a85b2dc4 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -19,9 +19,7 @@ #pragma once -#include #include -#include #include #include #include @@ -63,19 +61,10 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { file_manager_->NewOutputStream(file_name)); PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR bytes, writer_->SerializedBytes()); - uint64_t total_write_size = 0; - while (total_write_size < bytes->size()) { - uint64_t current_write_size = - std::min(bytes->size() - total_write_size, max_write_size_); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_size, - out->Write(bytes->data() + total_write_size, - static_cast(current_write_size))); - if (static_cast(actual_size) != current_write_size) { - return Status::IOError( - fmt::format("expect write len {} mismatch actual write len {}", - current_write_size, actual_size)); - } - total_write_size += current_write_size; + PAIMON_ASSIGN_OR_RAISE(int64_t actual_size, out->Write(bytes->data(), bytes->size())); + if (actual_size < 0 || static_cast(actual_size) != bytes->size()) { + return Status::IOError(fmt::format("expect write len {} mismatch actual write len {}", + bytes->size(), actual_size)); } PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); @@ -85,11 +74,8 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { } private: - static constexpr uint64_t kMaxWriteSize = std::numeric_limits::max(); - std::string index_type_; int64_t count_ = 0; - uint64_t max_write_size_ = kMaxWriteSize; std::shared_ptr file_manager_; std::shared_ptr writer_; }; diff --git a/src/paimon/common/io/buffered_input_stream.cpp b/src/paimon/common/io/buffered_input_stream.cpp index d2b50b73..e02620c0 100644 --- a/src/paimon/common/io/buffered_input_stream.cpp +++ b/src/paimon/common/io/buffered_input_stream.cpp @@ -25,13 +25,14 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/memory/bytes.h" namespace paimon { class MemoryPool; BufferedInputStream::BufferedInputStream(const std::shared_ptr& in, - int32_t buffer_size, MemoryPool* pool) + int64_t buffer_size, MemoryPool* pool) : buffer_size_(buffer_size), in_(in) { assert(buffer_size > 0); buffer_ = std::make_unique(buffer_size, pool); @@ -60,7 +61,7 @@ Status BufferedInputStream::Seek(int64_t offset, SeekOrigin origin) { const int64_t buf_start_abs = in_pos - count_; const int64_t buf_end_abs = in_pos; if (target_abs_offset >= buf_start_abs && target_abs_offset <= buf_end_abs) { - pos_ = static_cast(target_abs_offset - buf_start_abs); + pos_ = target_abs_offset - buf_start_abs; return Status::OK(); } } @@ -78,10 +79,11 @@ Result BufferedInputStream::GetPos() const { return in_pos - count_ + pos_; } -Result BufferedInputStream::Read(char* buffer, uint32_t size) { - uint32_t actual_read_len = 0; +Result BufferedInputStream::Read(char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + int64_t actual_read_len = 0; while (actual_read_len < size) { - PAIMON_ASSIGN_OR_RAISE(int32_t nread, + PAIMON_ASSIGN_OR_RAISE(int64_t nread, InnerRead(buffer + actual_read_len, size - actual_read_len)); assert(nread > 0); actual_read_len += nread; @@ -90,16 +92,16 @@ Result BufferedInputStream::Read(char* buffer, uint32_t size) { return actual_read_len; } -Result BufferedInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { +Result BufferedInputStream::Read(char* buffer, int64_t size, int64_t offset) { return Status::Invalid("BufferedInputStream does not support Read from offset"); } -void BufferedInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void BufferedInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { callback(Status::NotImplemented("BufferedInputStream do not support ReadAsync")); } -Result BufferedInputStream::Length() const { +Result BufferedInputStream::Length() const { return in_->Length(); } @@ -119,18 +121,21 @@ Status BufferedInputStream::Fill() { count_ = 0; PAIMON_ASSIGN_OR_RAISE(int64_t in_pos, in_->GetPos()); PAIMON_ASSIGN_OR_RAISE(int64_t length, in_->Length()); - int64_t left_to_read = std::min((length - in_pos), static_cast(buffer_size_)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in_->Read(buffer_->data(), left_to_read)); + int64_t left_to_read = std::min(length - in_pos, buffer_size_); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, in_->Read(buffer_->data(), left_to_read)); PAIMON_RETURN_NOT_OK(AssertReadLength(left_to_read, actual_read_len)); count_ = actual_read_len; return Status::OK(); } -Result BufferedInputStream::InnerRead(char* buffer, int32_t size) { +Result BufferedInputStream::InnerRead(char* buffer, int64_t size) { assert(size > 0); - int32_t avail = count_ - pos_; - if (avail <= 0) { - assert(avail == 0); + if (PAIMON_UNLIKELY(pos_ > count_)) { + return Status::Invalid(fmt::format( + "BufferedInputStream internal error: pos_ {} exceeds count_ {}", pos_, count_)); + } + int64_t avail = count_ - pos_; + if (avail == 0) { /* If the requested length is at least as large as the buffer, and if there is no mark/reset activity, do not bother to copy the bytes into the local buffer. In this way buffered streams will @@ -147,14 +152,14 @@ Result BufferedInputStream::InnerRead(char* buffer, int32_t size) { size)); } } - int32_t copy_length = std::min(avail, size); - memcpy(buffer, buffer_->data() + pos_, copy_length); + int64_t copy_length = std::min(avail, size); + memcpy(buffer, buffer_->data() + pos_, static_cast(copy_length)); pos_ += copy_length; return copy_length; } -Status BufferedInputStream::AssertReadLength(int32_t read_length, - int32_t actual_read_length) const { +Status BufferedInputStream::AssertReadLength(int64_t read_length, + int64_t actual_read_length) const { if (read_length != actual_read_length) { return Status::Invalid( fmt::format("assert read length failed: read length not match, read length {}, actual " diff --git a/src/paimon/common/io/byte_array_input_stream.cpp b/src/paimon/common/io/byte_array_input_stream.cpp index a88d6b82..06f0b0e2 100644 --- a/src/paimon/common/io/byte_array_input_stream.cpp +++ b/src/paimon/common/io/byte_array_input_stream.cpp @@ -26,9 +26,10 @@ #include "fmt/format.h" namespace paimon { -ByteArrayInputStream::ByteArrayInputStream(const char* buffer, uint64_t length) +ByteArrayInputStream::ByteArrayInputStream(const char* buffer, int64_t length) : buffer_(buffer), length_(length), position_(0) { assert(buffer_); + assert(length >= 0); } const char* ByteArrayInputStream::GetRawData() const { @@ -36,59 +37,60 @@ const char* ByteArrayInputStream::GetRawData() const { } Status ByteArrayInputStream::Seek(int64_t offset, SeekOrigin origin) { + int64_t new_position = 0; switch (origin) { case SeekOrigin::FS_SEEK_SET: { - position_ = offset; + new_position = offset; break; } case SeekOrigin::FS_SEEK_CUR: { - position_ += offset; + new_position = position_ + offset; break; } case SeekOrigin::FS_SEEK_END: { - PAIMON_ASSIGN_OR_RAISE(uint64_t length, Length()); - position_ = static_cast(length) + offset; + new_position = length_ + offset; break; } default: return Status::Invalid( "invalid SeekOrigin, only support FS_SEEK_SET, FS_SEEK_CUR, and FS_SEEK_END"); } - if (position_ < 0 || position_ > static_cast(length_)) { - return Status::Invalid( - fmt::format("invalid seek, after seek, current pos {}, length {}", position_, length_)); + if (new_position < 0 || new_position > length_) { + return Status::Invalid(fmt::format("invalid seek, after seek, current pos {}, length {}", + new_position, length_)); } + position_ = new_position; return Status::OK(); } -Result ByteArrayInputStream::Read(char* buffer, uint32_t size) { - if (position_ + static_cast(size) > static_cast(length_)) { +Result ByteArrayInputStream::Read(char* buffer, int64_t size) { + if (size < 0 || size > length_ - position_) { return Status::Invalid( fmt::format("ByteArrayInputStream assert boundary failed: need length {}, current " "position {}, exceed length {}", size, position_, length_)); } - memcpy(buffer, buffer_ + position_, size); + memcpy(buffer, buffer_ + position_, static_cast(size)); position_ += size; return size; } -Result ByteArrayInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { - if (offset + static_cast(size) > length_) { +Result ByteArrayInputStream::Read(char* buffer, int64_t size, int64_t offset) { + if (size < 0 || offset < 0 || offset > length_ || size > length_ - offset) { return Status::Invalid( - fmt::format("ByteArrayInputStream assert boundary failed: need length {}, read offset " - "{}, exceed length {}", + fmt::format("ByteArrayInputStream boundary check failed: read size {}, offset {}, " + "stream length {}", size, offset, length_)); } - memcpy(buffer, buffer_ + offset, size); + memcpy(buffer, buffer_ + offset, static_cast(size)); return size; } -void ByteArrayInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void ByteArrayInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { - Result read_size = Read(buffer, size, offset); + Result read_size = Read(buffer, size, offset); Status status = Status::OK(); - if (read_size.ok() && static_cast(read_size.value()) != size) { + if (read_size.ok() && read_size.value() != size) { status = Status::Invalid(fmt::format( "ByteArrayInputStream async read size {} != expected {}", read_size.value(), size)); } else if (!read_size.ok()) { diff --git a/src/paimon/common/io/cache/cache.cpp b/src/paimon/common/io/cache/cache.cpp new file mode 100644 index 00000000..37a26e91 --- /dev/null +++ b/src/paimon/common/io/cache/cache.cpp @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/cache/cache.h" + +#include + +namespace paimon { + +CacheValue::CacheValue(const MemorySegment& segment, CacheCallback callback) + : segment_(segment), callback_(std::move(callback)) {} + +CacheValue::~CacheValue() = default; + +const MemorySegment& CacheValue::GetSegment() const { + return segment_; +} + +void CacheValue::OnEvict(const std::shared_ptr& key) const { + if (callback_) { + callback_(key); + } +} + +bool CacheValue::operator==(const CacheValue& other) const { + if (this == &other) { + return true; + } + return segment_ == other.segment_; +} + +} // namespace paimon diff --git a/src/paimon/common/io/cache/cache_key.cpp b/src/paimon/common/io/cache/cache_key.cpp index 383bb205..c84c8f90 100644 --- a/src/paimon/common/io/cache/cache_key.cpp +++ b/src/paimon/common/io/cache/cache_key.cpp @@ -19,10 +19,67 @@ #include "paimon/common/io/cache/cache_key.h" namespace paimon { +namespace { + +class SnapshotLiveManifestEntriesCacheKey : public CacheKey { + public: + SnapshotLiveManifestEntriesCacheKey(const std::string& table_path, const std::string& branch, + int32_t bucket) + : CacheKey(CacheKind::SNAPSHOT_LIVE_MANIFEST), + table_path_(table_path), + branch_(branch), + bucket_(bucket) {} + + bool IsIndex() const override { + return false; + } + + bool Equals(const CacheKey& other) const override { + const auto* rhs = dynamic_cast(&other); + if (!rhs) { + return false; + } + return table_path_ == rhs->table_path_ && branch_ == rhs->branch_ && + bucket_ == rhs->bucket_ && GetKind() == rhs->GetKind(); + } + + size_t HashCode() const override { + size_t seed = 0; + seed ^= std::hash{}(table_path_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(branch_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(bucket_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(static_cast(GetKind())) + HASH_CONSTANT + + (seed << 6) + (seed >> 2); + return seed; + } + + private: + static constexpr uint64_t HASH_CONSTANT = 0x9e3779b97f4a7c15ULL; + + const std::string table_path_; + const std::string branch_; + const int32_t bucket_; +}; + +} // namespace std::shared_ptr CacheKey::ForPosition(const std::string& file_path, int64_t position, int32_t length, bool is_index) { - return std::make_shared(file_path, position, length, is_index); + return std::make_shared(file_path, position, length, is_index, + CacheKind::DEFAULT); +} + +std::shared_ptr CacheKey::ForKind(const std::string& file_path, int64_t position, + int32_t length, CacheKind kind) { + auto key = std::make_shared(file_path, position, length, + /*is_index=*/false, kind); + return key; +} + +std::shared_ptr CacheKey::ForSnapshotLiveManifestEntries(const std::string& table_path, + const std::string& branch, + int32_t bucket) { + return std::make_shared(table_path, branch, bucket); } bool PositionCacheKey::IsIndex() const { @@ -43,7 +100,7 @@ bool PositionCacheKey::Equals(const CacheKey& other) const { return false; } return file_path_ == rhs->file_path_ && position_ == rhs->position_ && - length_ == rhs->length_ && is_index_ == rhs->is_index_; + length_ == rhs->length_ && is_index_ == rhs->is_index_ && GetKind() == rhs->GetKind(); } size_t PositionCacheKey::HashCode() const { @@ -52,6 +109,8 @@ size_t PositionCacheKey::HashCode() const { seed ^= std::hash{}(position_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); seed ^= std::hash{}(length_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); seed ^= std::hash{}(is_index_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(static_cast(GetKind())) + HASH_CONSTANT + (seed << 6) + + (seed >> 2); return seed; } diff --git a/src/paimon/common/io/cache/cache_key.h b/src/paimon/common/io/cache/cache_key.h index 75f8cb82..988735d1 100644 --- a/src/paimon/common/io/cache/cache_key.h +++ b/src/paimon/common/io/cache/cache_key.h @@ -17,36 +17,24 @@ */ #pragma once + #include #include #include -#include -#include "paimon/visibility.h" +#include "paimon/cache/cache.h" namespace paimon { -class CacheValue; - -class PAIMON_EXPORT CacheKey { - public: - static std::shared_ptr ForPosition(const std::string& file_path, int64_t position, - int32_t length, bool is_index); - - public: - virtual ~CacheKey() = default; - - virtual bool IsIndex() const = 0; - - virtual bool Equals(const CacheKey& other) const = 0; - - virtual size_t HashCode() const = 0; -}; - class PositionCacheKey : public CacheKey { public: - PositionCacheKey(const std::string& file_path, int64_t position, int32_t length, bool is_index) - : file_path_(file_path), position_(position), length_(length), is_index_(is_index) {} + PositionCacheKey(const std::string& file_path, int64_t position, int32_t length, bool is_index, + CacheKind kind) + : CacheKey(kind), + file_path_(file_path), + position_(position), + length_(length), + is_index_(is_index) {} bool IsIndex() const override; size_t HashCode() const override; diff --git a/src/paimon/common/io/cache/cache_manager.h b/src/paimon/common/io/cache/cache_manager.h index bad3f68d..f899d46c 100644 --- a/src/paimon/common/io/cache/cache_manager.h +++ b/src/paimon/common/io/cache/cache_manager.h @@ -22,10 +22,10 @@ #include #include -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" #include "paimon/common/io/cache/lru_cache.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/io/cache/lru_cache.h b/src/paimon/common/io/cache/lru_cache.h index 07d6038b..745ca7f5 100644 --- a/src/paimon/common/io/cache/lru_cache.h +++ b/src/paimon/common/io/cache/lru_cache.h @@ -22,7 +22,7 @@ #include #include -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" #include "paimon/common/utils/generic_lru_cache.h" #include "paimon/result.h" diff --git a/src/paimon/common/io/cache/lru_cache_test.cpp b/src/paimon/common/io/cache/lru_cache_test.cpp index 9e8ae449..1d644c70 100644 --- a/src/paimon/common/io/cache/lru_cache_test.cpp +++ b/src/paimon/common/io/cache/lru_cache_test.cpp @@ -27,10 +27,10 @@ #include #include "gtest/gtest.h" -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -362,6 +362,44 @@ TEST_F(LruCacheTest, TestPutMovesToFront) { ASSERT_EQ(evicted[0], 1); } +TEST_F(LruCacheTest, TestForKindSetsKeyKind) { + LruCache cache(1024); + + auto key = CacheKey::ForKind("test_file", 0, 64, CacheKind::MANIFEST); + bool supplier_seen_kind = false; + auto supplier = + [&](const std::shared_ptr& supplier_key) -> Result> { + supplier_seen_kind = supplier_key->GetKind() == CacheKind::MANIFEST; + return MakeValue(64, 'M'); + }; + + ASSERT_OK_AND_ASSIGN(auto value, cache.Get(key, supplier)); + ASSERT_TRUE(supplier_seen_kind); + ASSERT_EQ(CacheKind::MANIFEST, key->GetKind()); + ASSERT_EQ('M', value->GetSegment().Get(0)); + + auto put_key = CacheKey::ForKind("test_file", 1, 64, CacheKind::MANIFEST); + ASSERT_OK(cache.Put(put_key, MakeValue(64, 'P'))); + ASSERT_EQ(CacheKind::MANIFEST, put_key->GetKind()); +} + +TEST_F(LruCacheTest, TestForSnapshotLiveManifestEntries) { + auto main_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 0); + auto same_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 0); + auto branch_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "dev", 0); + auto table_key = CacheKey::ForSnapshotLiveManifestEntries("other_table_path", "main", 0); + auto bucket_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 1); + auto hash_in_path_key = CacheKey::ForSnapshotLiveManifestEntries("table#path", "main", 0); + auto hash_in_branch_key = CacheKey::ForSnapshotLiveManifestEntries("table", "path#main", 0); + + ASSERT_EQ(CacheKind::SNAPSHOT_LIVE_MANIFEST, main_key->GetKind()); + ASSERT_TRUE(CacheKeyEqual()(main_key, same_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, branch_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, table_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, bucket_key)); + ASSERT_FALSE(CacheKeyEqual()(hash_in_path_key, hash_in_branch_key)); +} + /// Verifies that multiple evictions happen when a single large entry is inserted. TEST_F(LruCacheTest, TestMultipleEvictions) { LruCache cache(300); diff --git a/src/paimon/common/io/cache_input_stream.h b/src/paimon/common/io/cache_input_stream.h index 3c48df30..9ccbf260 100644 --- a/src/paimon/common/io/cache_input_stream.h +++ b/src/paimon/common/io/cache_input_stream.h @@ -22,6 +22,7 @@ #include #include +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/utils/read_ahead_cache.h" @@ -39,12 +40,14 @@ class CacheInputStream : public InputStream { Result GetPos() const override { return input_stream_->GetPos(); } - Result Read(char* buffer, uint32_t size) override { + Result Read(char* buffer, int64_t size) override { return input_stream_->Read(buffer, size); } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { + Result Read(char* buffer, int64_t size, int64_t offset) override { if (cache_) { - ByteRange range{offset, static_cast(size)}; + PAIMON_RETURN_NOT_OK(ValidateValueInRange(offset, "read offset")); + PAIMON_RETURN_NOT_OK(ValidateValueInRange(size, "read size")); + ByteRange range{static_cast(offset), static_cast(size)}; PAIMON_ASSIGN_OR_RAISE(ByteSlice slice, cache_->Read(range)); if (slice.buffer) { std::memcpy(buffer, slice.buffer->data() + slice.offset, slice.length); @@ -53,10 +56,20 @@ class CacheInputStream : public InputStream { } return input_stream_->Read(buffer, size, offset); } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { if (cache_) { - ByteRange range{offset, static_cast(size)}; + Status status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + callback(status); + return; + } + status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + callback(status); + return; + } + ByteRange range{static_cast(offset), static_cast(size)}; Result slice = cache_->Read(range); if (!slice.ok()) { callback(slice.status()); @@ -80,7 +93,7 @@ class CacheInputStream : public InputStream { return input_stream_->GetUri(); } - Result Length() const override { + Result Length() const override { return input_stream_->Length(); } diff --git a/src/paimon/common/io/cache_input_stream_test.cpp b/src/paimon/common/io/cache_input_stream_test.cpp index 2c2e36cf..d4a61854 100644 --- a/src/paimon/common/io/cache_input_stream_test.cpp +++ b/src/paimon/common/io/cache_input_stream_test.cpp @@ -80,7 +80,7 @@ TEST_F(CacheInputStreamTest, TestProxyMethods) { CacheInputStream stream(std::move(underlying), /*cache=*/nullptr); // Length - ASSERT_OK_AND_ASSIGN(uint64_t length, stream.Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, stream.Length()); ASSERT_EQ(length, content_.size()); // GetUri @@ -94,7 +94,7 @@ TEST_F(CacheInputStreamTest, TestProxyMethods) { // Read (sequential, no offset) std::string buffer(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 3)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 3)); ASSERT_EQ(bytes_read, 3); ASSERT_EQ(buffer, "fgh"); @@ -108,7 +108,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetNullCache) { CacheInputStream stream(std::move(underlying), /*cache=*/nullptr); std::string buffer(5, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); ASSERT_EQ(bytes_read, 5); ASSERT_EQ(buffer, "cdefg"); } @@ -121,7 +121,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetCacheHit) { CacheInputStream stream(std::move(underlying), cache); std::string buffer(5, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); ASSERT_EQ(bytes_read, 5); ASSERT_EQ(buffer, "cdefg"); } @@ -134,7 +134,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetCacheMiss) { CacheInputStream stream(std::move(underlying), cache); std::string buffer(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 3, /*offset=*/10)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 3, /*offset=*/10)); ASSERT_EQ(bytes_read, 3); ASSERT_EQ(buffer, "klm"); } diff --git a/src/paimon/common/io/data_input_stream.cpp b/src/paimon/common/io/data_input_stream.cpp index 63f83381..9ffe75b0 100644 --- a/src/paimon/common/io/data_input_stream.cpp +++ b/src/paimon/common/io/data_input_stream.cpp @@ -42,10 +42,10 @@ Status DataInputStream::Seek(int64_t offset) const { template Result DataInputStream::ReadValue() const { static_assert(std::is_trivially_copyable_v, "T must be trivially copyable"); - int32_t read_length = sizeof(T); + int64_t read_length = sizeof(T); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); T value; - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(reinterpret_cast(&value), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); if (NeedSwap()) { @@ -55,17 +55,17 @@ Result DataInputStream::ReadValue() const { } Status DataInputStream::ReadBytes(Bytes* bytes) const { - int32_t read_length = bytes->size(); + int64_t read_length = bytes->size(); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(bytes->data(), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); return Status::OK(); } -Status DataInputStream::Read(char* data, uint32_t size) const { +Status DataInputStream::Read(char* data, int64_t size) const { PAIMON_RETURN_NOT_OK(AssertBoundary(size)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, input_stream_->Read(data, size)); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(data, size)); PAIMON_RETURN_NOT_OK(AssertReadLength(size, actual_read_length)); return Status::OK(); } @@ -75,7 +75,7 @@ Result DataInputStream::ReadString() const { PAIMON_ASSIGN_OR_RAISE(read_length, ReadValue()); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); std::string value(read_length, '\0'); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(value.data(), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); return value; @@ -85,11 +85,11 @@ Result DataInputStream::GetPos() const { return input_stream_->GetPos(); } -Result DataInputStream::Length() const { +Result DataInputStream::Length() const { return input_stream_->Length(); } -Status DataInputStream::AssertReadLength(int32_t read_length, int32_t actual_read_length) const { +Status DataInputStream::AssertReadLength(int64_t read_length, int64_t actual_read_length) const { if (read_length != actual_read_length) { return Status::Invalid( fmt::format("assert read length failed: read length not match, read length {}, actual " @@ -99,15 +99,16 @@ Status DataInputStream::AssertReadLength(int32_t read_length, int32_t actual_rea return Status::OK(); } -Status DataInputStream::AssertBoundary(int32_t need_length) const { +Status DataInputStream::AssertBoundary(int64_t need_length) const { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(need_length, "DataInputStream need length")); // TODO(jinli.zjw): Store current_pos and file_length as member variables to reduce the overhead // of I/O calls. PAIMON_ASSIGN_OR_RAISE(int64_t pos, input_stream_->GetPos()); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream_->Length()); - if (pos + need_length > static_cast(length)) { + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream_->Length()); + if (pos < 0 || length < 0 || pos > length || need_length > length - pos) { return Status::Invalid( - fmt::format("DataInputStream assert boundary failed: need length {}, current position " - "{}, exceed length {}", + fmt::format("DataInputStream boundary check failed: read size {}, position {}, " + "stream length {}", need_length, pos, length)); } return Status::OK(); diff --git a/src/paimon/common/io/data_output_stream.cpp b/src/paimon/common/io/data_output_stream.cpp index 14d7463f..53825a05 100644 --- a/src/paimon/common/io/data_output_stream.cpp +++ b/src/paimon/common/io/data_output_stream.cpp @@ -30,8 +30,8 @@ DataOutputStream::DataOutputStream(const std::shared_ptr& output_s } Status DataOutputStream::WriteBytes(const std::shared_ptr& bytes) { - int32_t write_length = bytes->size(); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_write_length, + int64_t write_length = bytes->size(); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_length, output_stream_->Write(bytes->data(), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); @@ -40,14 +40,14 @@ Status DataOutputStream::WriteBytes(const std::shared_ptr& bytes) { Status DataOutputStream::WriteString(const std::string& value) { uint16_t write_length = value.size(); PAIMON_RETURN_NOT_OK(WriteValue(write_length)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_write_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_length, output_stream_->Write(value.data(), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); } -Status DataOutputStream::AssertWriteLength(int32_t write_length, - int32_t actual_write_length) const { +Status DataOutputStream::AssertWriteLength(int64_t write_length, + int64_t actual_write_length) const { if (write_length != actual_write_length) { return Status::Invalid(fmt::format( "assert write length failed: write length not match, write length {}, actual " diff --git a/src/paimon/common/io/data_output_stream.h b/src/paimon/common/io/data_output_stream.h index ace5976a..dfb6456b 100644 --- a/src/paimon/common/io/data_output_stream.h +++ b/src/paimon/common/io/data_output_stream.h @@ -48,9 +48,9 @@ class PAIMON_EXPORT DataOutputStream { if (NeedSwap()) { write_value = EndianSwapValue(value); } - int32_t write_length = sizeof(T); + int64_t write_length = sizeof(T); PAIMON_ASSIGN_OR_RAISE( - int32_t actual_write_length, + int64_t actual_write_length, output_stream_->Write(reinterpret_cast(&write_value), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); @@ -66,7 +66,7 @@ class PAIMON_EXPORT DataOutputStream { } private: - Status AssertWriteLength(int32_t write_length, int32_t actual_write_length) const; + Status AssertWriteLength(int64_t write_length, int64_t actual_write_length) const; bool NeedSwap() const; diff --git a/src/paimon/common/io/memory_segment_output_stream.cpp b/src/paimon/common/io/memory_segment_output_stream.cpp index 498b5abc..5355f72b 100644 --- a/src/paimon/common/io/memory_segment_output_stream.cpp +++ b/src/paimon/common/io/memory_segment_output_stream.cpp @@ -55,7 +55,9 @@ void MemorySegmentOutputStream::WriteString(const std::string& str) { void MemorySegmentOutputStream::Write(const char* data, uint32_t size) { auto bytes = std::make_shared(size, pool_.get()); - memcpy(bytes->data(), data, size); + if (size != 0) { + memcpy(bytes->data(), data, size); + } auto segment = MemorySegment::Wrap(bytes); Write(segment, 0, segment.Size()); } diff --git a/src/paimon/common/io/memory_segment_output_stream.h b/src/paimon/common/io/memory_segment_output_stream.h index dbe3816f..0986344e 100644 --- a/src/paimon/common/io/memory_segment_output_stream.h +++ b/src/paimon/common/io/memory_segment_output_stream.h @@ -26,9 +26,9 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" +#include "paimon/memory/memory_segment.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" diff --git a/src/paimon/common/io/offset_input_stream.cpp b/src/paimon/common/io/offset_input_stream.cpp index 69593cd1..2df88adc 100644 --- a/src/paimon/common/io/offset_input_stream.cpp +++ b/src/paimon/common/io/offset_input_stream.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/macros.h" namespace paimon { @@ -30,28 +31,29 @@ Result> OffsetInputStream::Create( if (PAIMON_UNLIKELY(wrapped == nullptr)) { return Status::Invalid("input stream is null pointer"); } - if (PAIMON_UNLIKELY(offset < 0)) { - return Status::Invalid(fmt::format("offset {} is less than 0", offset)); - } - if (PAIMON_UNLIKELY(length < -1)) { - return Status::Invalid(fmt::format("length {} is less than -1", length)); + PAIMON_ASSIGN_OR_RAISE(int64_t total_length, wrapped->Length()); + return Create(wrapped, length, offset, total_length); +} + +Result> OffsetInputStream::Create( + const std::shared_ptr& wrapped, int64_t length, int64_t offset, + int64_t total_length) { + if (PAIMON_UNLIKELY(wrapped == nullptr)) { + return Status::Invalid("input stream is null pointer"); } - PAIMON_ASSIGN_OR_RAISE(uint64_t total_length, wrapped->Length()); - if (PAIMON_UNLIKELY((uint64_t)offset > total_length)) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "offset")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(total_length, "total length")); + if (PAIMON_UNLIKELY(offset > total_length)) { return Status::Invalid( fmt::format("offset {} exceed total length {}", offset, total_length)); } - if (length == -1) { - // length == -1 means it's dynamic length, should read to the end - length = total_length - offset; - } - if (PAIMON_UNLIKELY((uint64_t)offset + (uint64_t)length > total_length)) { + if (PAIMON_UNLIKELY(length > total_length - offset)) { return Status::Invalid(fmt::format("offset {} + length {} exceed total length {}", offset, length, total_length)); } PAIMON_RETURN_NOT_OK(wrapped->Seek(offset, SeekOrigin::FS_SEEK_SET)); - return std::unique_ptr( - new OffsetInputStream(std::move(wrapped), length, offset)); + return std::unique_ptr(new OffsetInputStream(wrapped, length, offset)); } OffsetInputStream::OffsetInputStream(const std::shared_ptr& wrapped, int64_t length, @@ -59,42 +61,48 @@ OffsetInputStream::OffsetInputStream(const std::shared_ptr& wrapped : wrapped_(wrapped), length_(length), offset_(offset) {} Status OffsetInputStream::Seek(int64_t offset, SeekOrigin origin) { + int64_t new_position = 0; switch (origin) { case SeekOrigin::FS_SEEK_SET: { - inner_position_ = offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset_ + inner_position_, SeekOrigin::FS_SEEK_SET); + new_position = offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset_ + new_position, SeekOrigin::FS_SEEK_SET)); + break; } case SeekOrigin::FS_SEEK_CUR: { - inner_position_ += offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset, SeekOrigin::FS_SEEK_CUR); + new_position = inner_position_ + offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset, SeekOrigin::FS_SEEK_CUR)); + break; } case SeekOrigin::FS_SEEK_END: { - inner_position_ = length_ + offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset_ + inner_position_, SeekOrigin::FS_SEEK_SET); + new_position = length_ + offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset_ + new_position, SeekOrigin::FS_SEEK_SET)); + break; } default: return Status::Invalid( "invalid SeekOrigin, only support FS_SEEK_SET, FS_SEEK_CUR, and FS_SEEK_END"); } + inner_position_ = new_position; return Status::OK(); } -Result OffsetInputStream::Read(char* buffer, uint32_t size) { +Result OffsetInputStream::Read(char* buffer, int64_t size) { PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_ + size)); - inner_position_ += size; - return wrapped_->Read(buffer, size); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, wrapped_->Read(buffer, size)); + inner_position_ += actual_read_len; + return actual_read_len; } -Result OffsetInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { +Result OffsetInputStream::Read(char* buffer, int64_t size, int64_t offset) { PAIMON_RETURN_NOT_OK(AssertBoundary(offset)); PAIMON_RETURN_NOT_OK(AssertBoundary(offset + size)); return wrapped_->Read(buffer, size, offset_ + offset); } -void OffsetInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void OffsetInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { auto status = AssertBoundary(offset); if (!status.ok()) { @@ -121,11 +129,11 @@ Result OffsetInputStream::GetPos() const { return inner_position_; } -Result OffsetInputStream::Length() const { +Result OffsetInputStream::Length() const { return length_; } -Status OffsetInputStream::AssertBoundary(int32_t inner_pos) const { +Status OffsetInputStream::AssertBoundary(int64_t inner_pos) const { if (inner_pos < 0 || inner_pos > length_) { return Status::Invalid( fmt::format("OffsetInputStream assert boundary failed: inner pos {} exceed length {}", diff --git a/src/paimon/common/io/offset_input_stream.h b/src/paimon/common/io/offset_input_stream.h index 38f97ef9..9fb6f4c0 100644 --- a/src/paimon/common/io/offset_input_stream.h +++ b/src/paimon/common/io/offset_input_stream.h @@ -35,20 +35,23 @@ class PAIMON_EXPORT OffsetInputStream : public InputStream { public: static Result> Create( const std::shared_ptr& wrapped, int64_t length, int64_t offset); + static Result> Create( + const std::shared_ptr& wrapped, int64_t length, int64_t offset, + int64_t total_length); ~OffsetInputStream() override = default; Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override; + Result Length() const override; Status Close() override; @@ -56,7 +59,7 @@ class PAIMON_EXPORT OffsetInputStream : public InputStream { private: OffsetInputStream(const std::shared_ptr& wrapped, int64_t length, int64_t offset); - Status AssertBoundary(int32_t inner_pos) const; + Status AssertBoundary(int64_t inner_pos) const; private: std::shared_ptr wrapped_; diff --git a/src/paimon/common/io/offset_input_stream_test.cpp b/src/paimon/common/io/offset_input_stream_test.cpp index 3734650b..734c814f 100644 --- a/src/paimon/common/io/offset_input_stream_test.cpp +++ b/src/paimon/common/io/offset_input_stream_test.cpp @@ -146,43 +146,6 @@ TEST(OffsetInputStreamTest, TestBoundaryValidation) { "assert boundary failed: inner pos 9 exceed length 6"); } -TEST(OffsetInputStreamTest, TestReadWithUnspecifiedLength) { - auto inner_stream = std::make_unique("abcdefghij", /*length=*/10); - // Use -1 for length to test dynamic length calculation - ASSERT_OK_AND_ASSIGN( - auto offset_stream, - OffsetInputStream::Create(std::move(inner_stream), /*length=*/-1, /*offset=*/2)); - - // Test that length is calculated correctly - ASSERT_OK_AND_ASSIGN(auto length, offset_stream->Length()); - // Should be total length (10) minus offset (2) = 8 - ASSERT_EQ(8, length); - - // Test sequential read within the calculated bounds - std::string buffer(4, '\0'); - ASSERT_OK_AND_ASSIGN(auto bytes_read, offset_stream->Read(buffer.data(), /*size=*/4)); - ASSERT_EQ(4, bytes_read); - ASSERT_EQ("cdef", buffer); - - ASSERT_OK_AND_ASSIGN(auto pos, offset_stream->GetPos()); - ASSERT_EQ(4, pos); - - // Test read with offset within the calculated bounds - std::string buffer2(3, '\0'); - ASSERT_OK_AND_ASSIGN(bytes_read, offset_stream->Read(buffer2.data(), /*size=*/3, /*offset=*/5)); - ASSERT_EQ(3, bytes_read); - ASSERT_EQ("hij", buffer2); - - // Position should not change after offset read - ASSERT_OK_AND_ASSIGN(pos, offset_stream->GetPos()); - ASSERT_EQ(4, pos); - - // Test boundary validation with dynamic length - std::string buffer3(10, '\0'); - ASSERT_NOK_WITH_MSG(offset_stream->Read(buffer3.data(), /*size=*/10), - "assert boundary failed: inner pos 14 exceed length 8"); -} - TEST(OffsetInputStreamTest, TestInvalidParameters) { // Test null wrapped stream ASSERT_NOK_WITH_MSG(OffsetInputStream::Create(nullptr, /*length=*/6, /*offset=*/2), @@ -194,11 +157,11 @@ TEST(OffsetInputStreamTest, TestInvalidParameters) { OffsetInputStream::Create(std::move(inner_stream), /*length=*/6, /*offset=*/-1), "offset -1 is less than 0"); - // Test length less than -1 + // Test negative length inner_stream = std::make_unique("abcdefghij", /*length=*/10); ASSERT_NOK_WITH_MSG( OffsetInputStream::Create(std::move(inner_stream), /*length=*/-2, /*offset=*/2), - "length -2 is less than -1"); + "length -2 is less than 0"); // Test length + offset beyond wrapped stream length inner_stream = std::make_unique("abcdefghij", /*length=*/10); @@ -206,10 +169,10 @@ TEST(OffsetInputStreamTest, TestInvalidParameters) { OffsetInputStream::Create(std::move(inner_stream), /*length=*/8, /*offset=*/7), "offset 7 + length 8 exceed total length 10"); - // Test dynamic length with offset beyond wrapped stream length + // Test offset beyond wrapped stream length inner_stream = std::make_unique("abcdefghij", /*length=*/10); ASSERT_NOK_WITH_MSG( - OffsetInputStream::Create(std::move(inner_stream), /*length=*/-1, /*offset=*/15), + OffsetInputStream::Create(std::move(inner_stream), /*length=*/1, /*offset=*/15), "offset 15 exceed total length 10"); } diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp index 5e8a3f18..252e7a52 100644 --- a/src/paimon/common/logging/logging.cpp +++ b/src/paimon/common/logging/logging.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,16 @@ namespace paimon { +static const char* GetProgramName() { +#if defined(__APPLE__) + return getprogname(); +#elif defined(__GLIBC__) + return program_invocation_name; +#else + return "paimon-cpp"; +#endif +} + static std::optional& getLoggerCreator() { static std::optional _loggerCreator; return _loggerCreator; @@ -86,7 +97,7 @@ std::unique_ptr Logger::GetLogger(const std::string& path) { } std::unique_lock ulock(getRegistryLock()); if (!google::IsGoogleLoggingInitialized()) { - google::InitGoogleLogging(program_invocation_name); + google::InitGoogleLogging(GetProgramName()); } return std::make_unique(); } diff --git a/src/paimon/common/logging/logging_test.cpp b/src/paimon/common/logging/logging_test.cpp index 8b6e3f69..6ffec1f8 100644 --- a/src/paimon/common/logging/logging_test.cpp +++ b/src/paimon/common/logging/logging_test.cpp @@ -18,16 +18,61 @@ */ #include "paimon/logging.h" +#include +#include #include +#include +#include #include +#include #include +#include +#include "glog/log_severity.h" +#include "glog/logging.h" +#include "glog/raw_logging.h" #include "paimon/common/executor/future.h" #include "paimon/executor.h" +#include "paimon/fs/file_system.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +std::atomic g_creator_calls{0}; + +// A Logger that forwards to glog exactly like the built-in adaptor. It is used to +// restore behavior-preserving logging after exercising the custom-creator path, +// because the registry cannot be reset to "unset" through the public API. +class GlogForwardingLogger : public Logger { + public: + void LogV(PaimonLogLevel level, const char* fname, int lineno, const char* /*function*/, + const char* fmt, ...) override { + va_list args; + va_start(args, fmt); + google::RawLog__(ToGlog(level), fname, lineno, fmt, args); + va_end(args); + } + + bool IsLevelEnabled(PaimonLogLevel /*level*/) const override { + return true; + } + + private: + static google::LogSeverity ToGlog(PaimonLogLevel level) { + switch (level) { + case PAIMON_LOG_LEVEL_WARN: + return google::GLOG_WARNING; + case PAIMON_LOG_LEVEL_ERROR: + return google::GLOG_ERROR; + default: + return google::GLOG_INFO; + } + } +}; + +} // namespace TEST(LoggerTest, TestMultiThreadGetLogger) { - auto executor = CreateDefaultExecutor(/*thread_count=*/4); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4)); auto get_logger = []() { auto logger = Logger::GetLogger("my_log"); ASSERT_TRUE(logger); @@ -39,4 +84,101 @@ TEST(LoggerTest, TestMultiThreadGetLogger) { } Wait(futures); } + +TEST(LoggerTest, TestLogAllSeverities) { + // The default logger routes every PaimonLogLevel through the severity mapping. + auto logger = Logger::GetLogger("severity_test"); + ASSERT_TRUE(logger); + + logger->LogV(PAIMON_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __FUNCTION__, "debug severity"); + logger->LogV(PAIMON_LOG_LEVEL_INFO, __FILE__, __LINE__, __FUNCTION__, "info severity"); + logger->LogV(PAIMON_LOG_LEVEL_WARN, __FILE__, __LINE__, __FUNCTION__, "warn severity"); + logger->LogV(PAIMON_LOG_LEVEL_ERROR, __FILE__, __LINE__, __FUNCTION__, "error severity"); + // NONE and MAX fall into the default branch of the mapping. + logger->LogV(PAIMON_LOG_LEVEL_NONE, __FILE__, __LINE__, __FUNCTION__, "none severity"); + logger->LogV(PAIMON_LOG_LEVEL_MAX, __FILE__, __LINE__, __FUNCTION__, "max severity"); +} + +// Demonstrates that glog's normal LOG() path actually writes a log file to disk. +// Note: Paimon's Logger/GlogAdaptor uses google::RawLog__, which only goes to +// stderr and never touches disk, so this test drives glog's file sink directly. +TEST(LoggerTest, TestGlogWritesLogFileToDisk) { + // Save the global glog flags we are about to change so other tests are unaffected. + const bool prev_logtostderr = FLAGS_logtostderr; + const bool prev_timestamp_in_name = FLAGS_timestamp_in_logfile_name; + const int32_t prev_minloglevel = FLAGS_minloglevel; + + // A unique, empty directory so the only file inside is the one glog creates for us. + // The directory (and everything glog wrote into it) is removed on destruction. + auto tmp_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + std::shared_ptr fs = tmp_dir->GetFileSystem(); + const std::string base = tmp_dir->Str() + "/paimon_demo"; + + FLAGS_logtostderr = false; // must be false, otherwise glog skips the file sink + FLAGS_timestamp_in_logfile_name = false; // deterministic file name (no time/pid suffix) + FLAGS_minloglevel = google::GLOG_INFO; // do not filter out INFO + + if (!google::IsGoogleLoggingInitialized()) { + google::InitGoogleLogging("paimon-log-disk-test"); + } + // Force INFO logs to a file under our unique directory. Disable the + // ".INFO" symlink glog normally creates next to the log file: it would + // dangle during recursive deletion and break UniqueTestDirectory cleanup. + google::SetLogDestination(google::GLOG_INFO, base.c_str()); + google::SetLogSymlink(google::GLOG_INFO, ""); + + const std::string token = "PAIMON_DISK_LOG_DEMO_" + std::to_string(RandomNumber(0, 1'000'000)); + LOG(INFO) << "hello from disk logging, token=" << token; + google::FlushLogFiles(google::GLOG_INFO); + + // Collect the content of whatever file glog created in our directory. + std::string on_disk_path; + std::string content; + std::vector> entries; + ASSERT_OK(fs->ListDir(tmp_dir->Str(), &entries)); + for (const auto& entry : entries) { + if (entry->IsDir()) { + continue; + } + std::string file_content; + ASSERT_OK(fs->ReadFile(entry->GetPath(), &file_content)); + if (file_content.find(token) != std::string::npos) { + on_disk_path = entry->GetPath(); + content = std::move(file_content); + break; + } + } + + // Show the real on-disk file and its raw content so it can be eyeballed in test output. + std::cout << "\n===== on-disk glog file: " << on_disk_path << " =====\n" + << content << "===== end of file =====\n"; + + ASSERT_FALSE(on_disk_path.empty()) + << "no glog file containing the token was written to " << tmp_dir->Str(); + ASSERT_NE(content.find(token), std::string::npos); + + // Stop writing INFO logs to the directory that is deleted when tmp_dir goes out of + // scope, then restore flags. + google::SetLogDestination(google::GLOG_INFO, ""); + FLAGS_logtostderr = prev_logtostderr; + FLAGS_timestamp_in_logfile_name = prev_timestamp_in_name; + FLAGS_minloglevel = prev_minloglevel; +} + +// Keep this test last: it installs a process-wide logger creator that cannot be +// unset, so it must not run before tests that rely on the default logger. +TEST(LoggerTest, TestRegisterCustomLoggerCreator) { + g_creator_calls.store(0); + Logger::RegisterLogger([](const std::string& /*path*/) -> std::unique_ptr { + g_creator_calls.fetch_add(1); + return std::make_unique(); + }); + + auto logger = Logger::GetLogger("custom_path"); + ASSERT_TRUE(logger); + // GetLogger must have gone through the registered creator branch. + ASSERT_EQ(1, g_creator_calls.load()); + ASSERT_TRUE(logger->IsLevelEnabled(PAIMON_LOG_LEVEL_INFO)); +} } // namespace paimon::test diff --git a/src/paimon/common/memory/bytes.cpp b/src/paimon/common/memory/bytes.cpp index bfe7b28e..d1093f52 100644 --- a/src/paimon/common/memory/bytes.cpp +++ b/src/paimon/common/memory/bytes.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -33,7 +32,7 @@ const std::shared_ptr& Bytes::EmptyBytes() { return empty_bytes; } -PAIMON_UNIQUE_PTR Bytes::AllocateBytes(int32_t length, MemoryPool* pool) { +PAIMON_UNIQUE_PTR Bytes::AllocateBytes(size_t length, MemoryPool* pool) { return pool->AllocateUnique(length, pool); } diff --git a/src/paimon/common/memory/memory_segment.cpp b/src/paimon/common/memory/memory_segment.cpp index be4a8d5f..5e4cb8df 100644 --- a/src/paimon/common/memory/memory_segment.cpp +++ b/src/paimon/common/memory/memory_segment.cpp @@ -17,12 +17,22 @@ * under the License. */ -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include +#include "paimon/common/utils/math.h" + namespace paimon { +uint64_t MemorySegment::GetLongBigEndian(int32_t index) const { + auto value = GetValue(index); + if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) { + return EndianSwapValue(value); + } + return value; +} + int32_t MemorySegment::Compare(const MemorySegment& seg2, int32_t offset1, int32_t offset2, int32_t len) const { while (len >= 8) { diff --git a/src/paimon/common/memory/memory_segment_test.cpp b/src/paimon/common/memory/memory_segment_test.cpp index 05012950..ec583044 100644 --- a/src/paimon/common/memory/memory_segment_test.cpp +++ b/src/paimon/common/memory/memory_segment_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include #include @@ -558,10 +558,7 @@ TEST(MemorySegmentTest, TestDoubleAccess) { delete[] occupied; } -// ------------------------------------------------------------------------ -// Bulk Byte Movements -// ------------------------------------------------------------------------ - +// Bulk Byte Movements TEST(MemorySegmentTest, TestBulkByteAccess) { auto pool = paimon::GetDefaultPool(); // test expected correct behavior with default offset / length diff --git a/src/paimon/common/memory/memory_segment_utils.cpp b/src/paimon/common/memory/memory_segment_utils.cpp index 15c05455..fe406f9a 100644 --- a/src/paimon/common/memory/memory_segment_utils.cpp +++ b/src/paimon/common/memory/memory_segment_utils.cpp @@ -24,9 +24,6 @@ #include "paimon/common/utils/murmurhash_utils.h" namespace paimon { -std::shared_ptr MemorySegmentUtils::AllocateBytes(int32_t length, MemoryPool* pool) { - return Bytes::AllocateBytes(length, pool); -} void MemorySegmentUtils::CopyFromBytes(std::vector* segments, int32_t offset, const Bytes& bytes, int32_t bytes_offset, @@ -240,8 +237,7 @@ bool MemorySegmentUtils::EqualsMultiSegments(const std::vector& s int32_t seg_offset2 = offset2 - seg_size2 * seg_index2; // equal to % while (len > 0) { - int32_t equal_len = - std::min(std::min(len, seg_size1 - seg_offset1), seg_size2 - seg_offset2); + int32_t equal_len = std::min({len, seg_size1 - seg_offset1, seg_size2 - seg_offset2}); if (!segments1[seg_index1].EqualTo(segments2[seg_index2], seg_offset1, seg_offset2, equal_len)) { return false; @@ -317,7 +313,7 @@ int32_t MemorySegmentUtils::HashByWords(const std::vector& segmen int32_t MemorySegmentUtils::HashMultiSegByWords(const std::vector& segments, int32_t offset, int32_t num_bytes, MemoryPool* pool) { - std::shared_ptr bytes = AllocateBytes(num_bytes, pool); + std::shared_ptr bytes = Bytes::AllocateBytes(num_bytes, pool); CopyMultiSegmentsToBytes(segments, offset, bytes.get(), 0, num_bytes); return MurmurHashUtils::HashUnsafeBytesByWords(reinterpret_cast(bytes->data()), 0, num_bytes); @@ -325,7 +321,7 @@ int32_t MemorySegmentUtils::HashMultiSegByWords(const std::vector int32_t MemorySegmentUtils::HashMultiSeg(const std::vector& segments, int32_t offset, int32_t num_bytes, MemoryPool* pool) { - std::shared_ptr bytes = AllocateBytes(num_bytes, pool); + std::shared_ptr bytes = Bytes::AllocateBytes(num_bytes, pool); CopyMultiSegmentsToBytes(segments, offset, bytes.get(), 0, num_bytes); return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(bytes->data()), 0, num_bytes); diff --git a/src/paimon/common/memory/memory_segment_utils.h b/src/paimon/common/memory/memory_segment_utils.h index b904728f..19785ec6 100644 --- a/src/paimon/common/memory/memory_segment_utils.h +++ b/src/paimon/common/memory/memory_segment_utils.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -28,10 +29,10 @@ #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -43,9 +44,6 @@ class PAIMON_EXPORT MemorySegmentUtils { MemorySegmentUtils() = delete; ~MemorySegmentUtils() = delete; - /// Allocate bytes in pool - static std::shared_ptr AllocateBytes(int32_t length, MemoryPool* pool); - /// Copy target segments from source byte[]. /// /// @param segments target segments. diff --git a/src/paimon/common/memory/memory_slice.h b/src/paimon/common/memory/memory_slice.h index 2f45683f..4ed9f385 100644 --- a/src/paimon/common/memory/memory_slice.h +++ b/src/paimon/common/memory/memory_slice.h @@ -24,8 +24,8 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/predicate/literal.cpp b/src/paimon/common/predicate/literal.cpp index 0157a38b..3b2bcc0e 100644 --- a/src/paimon/common/predicate/literal.cpp +++ b/src/paimon/common/predicate/literal.cpp @@ -158,7 +158,9 @@ Literal::Literal(FieldType binary_type, const char* str, size_t size, bool own_d impl_->own_data_ = own_data; if (own_data) { impl_->value_.Buffer = new char[size]; - memcpy(impl_->value_.Buffer, str, size); + if (size > 0) { + memcpy(impl_->value_.Buffer, str, size); + } impl_->hash_code_ = impl_->CalculateHashCode(); } else { impl_->value_.Buffer = const_cast(str); @@ -221,7 +223,9 @@ Literal& Literal::operator=(const Literal& other) { impl_->type_ == FieldType::BLOB) && impl_->own_data_) { impl_->value_.Buffer = new char[other.impl_->size_]; - memcpy(impl_->value_.Buffer, other.impl_->value_.Buffer, other.impl_->size_); + if (other.impl_->size_ > 0) { + memcpy(impl_->value_.Buffer, other.impl_->value_.Buffer, other.impl_->size_); + } } else { impl_->value_ = other.impl_->value_; } diff --git a/src/paimon/common/predicate/predicate.cpp b/src/paimon/common/predicate/predicate.cpp new file mode 100644 index 00000000..6d902080 --- /dev/null +++ b/src/paimon/common/predicate/predicate.cpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/predicate/predicate.h" + +namespace paimon { +/// key function technique for Predicate +Predicate::~Predicate() = default; +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp new file mode 100644 index 00000000..68d45c11 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> BlobFallbackBatchReader::Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool) { + if (sequence_groups.size() < 2) { + return Status::Invalid( + "Blob fallback needs at least two sequence groups; a single group should be read " + "sequentially."); + } + if (read_schema == nullptr) { + return Status::Invalid("Blob fallback read schema cannot be nullptr."); + } + if (read_batch_size <= 0) { + return Status::Invalid(fmt::format( + "Blob fallback read batch size '{}' should be larger than zero", read_batch_size)); + } + int32_t blob_field_idx = -1; + for (int32_t i = 0; i < read_schema->num_fields(); i++) { + if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (blob_field_idx != -1) { + return Status::Invalid( + "Blob fallback read schema should contain exactly one blob field."); + } + blob_field_idx = i; + } + } + if (blob_field_idx == -1) { + return Status::Invalid("Blob fallback read schema should contain a blob field."); + } + int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name()); + int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); + std::vector groups; + groups.reserve(sequence_groups.size()); + for (auto& segments : sequence_groups) { + if (segments.empty()) { + return Status::Invalid("Blob fallback sequence group should not be empty."); + } + for (const auto& segment : segments) { + if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) { + return Status::Invalid( + "Blob fallback gap segment should cover at least one selected row id."); + } + } + GroupCursor cursor; + cursor.segments = std::move(segments); + groups.push_back(std::move(cursor)); + } + return std::unique_ptr( + new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx, + row_id_field_idx, seq_num_field_idx, read_batch_size, pool)); +} + +BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool) + : groups_(std::move(groups)), + read_schema_(read_schema), + blob_field_idx_(blob_field_idx), + row_id_field_idx_(row_id_field_idx), + seq_num_field_idx_(seq_num_field_idx), + read_batch_size_(read_batch_size), + arrow_pool_(GetArrowPool(pool)) {} + +Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want, + std::vector* chunks) { + GroupCursor& cursor = groups_[group_idx]; + int64_t collected = 0; + while (collected < want) { + if (!cursor.pending.empty()) { + const std::shared_ptr& front = cursor.pending.front(); + int64_t available = front->length() - cursor.pending_pos; + int64_t take = std::min(available, want - collected); + chunks->push_back(Chunk{front, cursor.pending_pos, take, {}}); + cursor.pending_pos += take; + collected += take; + if (cursor.pending_pos == front->length()) { + cursor.pending.pop_front(); + cursor.pending_pos = 0; + } + continue; + } + if (cursor.segment_idx >= cursor.segments.size()) { + // group exhausted; only the first group may define a shorter window + break; + } + Segment& segment = cursor.segments[cursor.segment_idx]; + if (segment.reader == nullptr) { + // gap segment: all rows are placeholders, stepped range by range so the row ids + // stay available for all-placeholder rows + if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) { + cursor.segment_idx++; + cursor.gap_range_idx = 0; + cursor.gap_range_pos = 0; + continue; + } + const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx]; + int64_t remaining = range.Count() - cursor.gap_range_pos; + int64_t take = std::min(remaining, want - collected); + Chunk chunk{nullptr, 0, take, {}}; + if (row_id_field_idx_ >= 0) { + chunk.gap_row_ids.reserve(take); + for (int64_t k = 0; k < take; k++) { + chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k); + } + } + chunks->push_back(std::move(chunk)); + cursor.gap_range_pos += take; + collected += take; + if (cursor.gap_range_pos == range.Count()) { + cursor.gap_range_idx++; + cursor.gap_range_pos = 0; + } + continue; + } + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, + segment.reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + cursor.segment_idx++; + continue; + } + auto& [read_batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = read_batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec, + ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap)); + for (const auto& selected_array : selected_array_vec) { + if (selected_array->length() == 0) { + continue; + } + auto struct_array = std::dynamic_pointer_cast(selected_array); + if (struct_array == nullptr) { + return Status::Invalid("Blob fallback expects file readers to emit struct arrays."); + } + cursor.pending.push_back(std::move(struct_array)); + } + } + return collected; +} + +Result> BlobFallbackBatchReader::ComputePlaceholderFlags( + const std::vector& chunks, int64_t row_count) const { + std::vector flags(row_count, false); + int64_t pos = 0; + for (const auto& chunk : chunks) { + if (chunk.array == nullptr) { + // gap rows stand for placeholders + std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); + } else { + std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); + auto binary_col = std::dynamic_pointer_cast(blob_col); + if (binary_col == nullptr) { + return Status::Invalid(fmt::format( + "Blob fallback expects the blob column to be large binary, but got {}", + blob_col->type()->ToString())); + } + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (binary_col->IsNull(idx)) { + continue; + } + std::string_view value = binary_col->GetView(idx); + if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) { + flags[pos + k] = true; + } + } + } + pos += chunk.length; + } + return flags; +} + +Result> BlobFallbackBatchReader::AssembleRowIdRun( + const std::vector& chunks, int64_t run_start, int64_t run_end) const { + arrow::ArrayVector pieces; + int64_t pos = 0; + for (const auto& chunk : chunks) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array != nullptr) { + std::shared_ptr column = chunk.array->field(row_id_field_idx_); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } else { + arrow::Int64Builder builder(arrow_pool_.get()); + for (int64_t r = overlap_start; r < overlap_end; r++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos])); + } + std::shared_ptr piece; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece)); + pieces.push_back(std::move(piece)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result> BlobFallbackBatchReader::AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const { + const auto row_count = static_cast(group_choice.size()); + arrow::ArrayVector pieces; + int64_t run_start = 0; + while (run_start < row_count) { + const int32_t group = group_choice[run_start]; + int64_t run_end = run_start + 1; + while (run_end < row_count && group_choice[run_end] == group) { + run_end++; + } + if (group < 0) { + // placeholder in every layer: the blob degrades to null; the row keeps its row id + // (taken from the newest group, which steps in lockstep), reports -1 as its + // sequence number, and returns null for every other field + if (field_idx == row_id_field_idx_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece, + AssembleRowIdRun(group_chunks[0], run_start, run_end)); + pieces.push_back(std::move(row_id_piece)); + } else if (field_idx == seq_num_field_idx_) { + arrow::Int64Scalar seq_scalar(-1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr seq_piece, + arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(seq_piece)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_piece, + arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(), + run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(null_piece)); + } + } else { + int64_t pos = 0; + for (const auto& chunk : group_chunks[group]) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array == nullptr) { + return Status::Invalid( + "Unexpected: a gap row was chosen as a blob fallback result."); + } + std::shared_ptr column = chunk.array->field(field_idx); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + } + run_start = run_end; + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the + // zero-offset BatchReader contract. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result BlobFallbackBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("blob fallback batch reader is closed"); + } + std::vector> group_chunks(groups_.size()); + // the first (newest) group defines the window; the others must step in lockstep + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0])); + for (size_t g = 1; g < groups_.size(); g++) { + // ask for one row even when the first group is exhausted, so that a longer group is + // reported as a misalignment instead of silently truncating the read + const int64_t want = std::max(row_count, 1); + PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, want, &group_chunks[g])); + if (got != row_count) { + return Status::Invalid(fmt::format( + "All sequence groups of a blob fallback read should have the same number of " + "rows: group {} yielded {} rows in a window of {}", + g, got, row_count)); + } + } + if (row_count == 0) { + return BatchReader::MakeEofBatch(); + } + + std::vector> placeholder_flags(groups_.size()); + for (size_t g = 0; g < groups_.size(); g++) { + PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g], + ComputePlaceholderFlags(group_chunks[g], row_count)); + } + // per row, the first group in max-sequence order with a real entry wins; -1 means the row is + // a placeholder in every group + std::vector group_choice(row_count, -1); + for (int64_t r = 0; r < row_count; r++) { + for (size_t g = 0; g < groups_.size(); g++) { + if (!placeholder_flags[g][r]) { + group_choice[r] = static_cast(g); + break; + } + } + } + + arrow::ArrayVector columns; + columns.reserve(read_schema_->num_fields()); + for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column, + AssembleColumn(field_idx, group_choice, group_chunks)); + columns.push_back(std::move(column)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(columns, read_schema_->fields())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); +} + +Result BlobFallbackBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return BatchReader::MakeEofBatchWithBitmap(); + } + return ReaderUtils::AddAllValidBitmap(std::move(batch)); +} + +void BlobFallbackBatchReader::Close() { + for (auto& group : groups_) { + group.pending.clear(); + for (auto& segment : group.segments) { + if (segment.reader) { + segment.reader->Close(); + } + } + } + closed_ = true; +} + +std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const { + auto metrics = std::make_shared(); + for (const auto& group : groups_) { + for (const auto& segment : group.segments) { + if (segment.reader) { + auto reader_metrics = segment.reader->GetReaderMetrics(); + if (reader_metrics) { + metrics->Merge(reader_metrics); + } + } + } + } + return metrics; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h new file mode 100644 index 00000000..e265c280 --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" + +namespace arrow { +class Array; +class StructArray; +} // namespace arrow + +namespace paimon { + +/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence +/// number layers, resolving placeholder entries row by row. Aligned with Java's +/// BlobFallbackRecordReader / AllPlaceholdersRecordReader. +/// +/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob +/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the +/// value from the newest layer that holds a real (non-placeholder) entry: +/// +/// 1. The caller groups the blob files by max sequence number (one group per layer, newest +/// first) and, inside each group, orders them by first row id. Row id ranges the group's +/// files do not cover are represented by gap segments, which stand for all-placeholder rows. +/// 2. All groups span the same overall row id range, so with the same row-ranges selection +/// applied they yield the same number of rows and can be stepped in lockstep. +/// 3. Each output row takes the first group, in max-sequence order, whose row is not a +/// placeholder. Placeholder rows are identified by exact equality with the +/// BlobDefs::kPlaceholderSentinel bytes, emitted by the blob format reader when +/// BlobDefs::kEmitPlaceholderSentinelKey is set. +/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its +/// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. +class BlobFallbackBatchReader : public BatchReader { + public: + /// One piece of a sequence group: either a reader over a single blob file (already wrapped + /// with the usual per-file mapping readers and row selection), or a virtual gap standing for + /// row ids the group's files do not cover. Segments must be ordered by ascending row id. + struct Segment { + /// File segment: emits the rows of one blob file. Null for a gap segment. + std::unique_ptr reader; + /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted + /// disjoint ranges. Must not be empty for a gap segment. + std::vector gap_selected_ranges; + }; + + /// `sequence_groups` must be ordered by descending max sequence number and contain at least + /// two groups (a single group needs no fallback). `read_schema` is the schema every file + /// reader emits; it must contain exactly one blob field and may additionally contain the + /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by + /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a + /// placeholder in every layer. + static Result> Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool); + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + void Close() override; + + std::shared_ptr GetReaderMetrics() const override; + + private: + /// A run of consecutive rows already fetched from one group: `array` rows + /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap + /// segment (array is null). + struct Chunk { + std::shared_ptr array; + int64_t offset = 0; + int64_t length = 0; + /// Gap chunk only: the row id of each of the `length` rows, filled when the read + /// schema contains _ROW_ID so all-placeholder rows can keep their row id. + std::vector gap_row_ids; + }; + + /// Read progress of one sequence group. Move-only, matching Segment's unique_ptr member. + struct GroupCursor { + GroupCursor() = default; + GroupCursor(const GroupCursor&) = delete; + GroupCursor& operator=(const GroupCursor&) = delete; + GroupCursor(GroupCursor&&) = default; + GroupCursor& operator=(GroupCursor&&) = default; + + std::vector segments; + size_t segment_idx = 0; + /// Position inside the current gap segment: index into gap_selected_ranges and the + /// number of rows already emitted from that range. + size_t gap_range_idx = 0; + int64_t gap_range_pos = 0; + /// Rows fetched from the current file segment but not yet consumed. + std::deque> pending; + int64_t pending_pos = 0; + }; + + BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool); + + /// Collects up to `want` rows from the group into chunks. Only the first group may come up + /// short (which defines the window size); any later group ending early is a misalignment. + Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks); + + /// Flags each of the `row_count` window rows of the given chunks as placeholder or not. + Result> ComputePlaceholderFlags(const std::vector& chunks, + int64_t row_count) const; + + /// Assembles one output column by stitching, per run of rows choosing the same group, + /// slices of that group's chunks. For rows choosing no group (placeholder in every layer), + /// _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and every other field becomes null. + Result> AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const; + + /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks; + /// gap chunks contribute their synthesized row ids. + Result> AssembleRowIdRun(const std::vector& chunks, + int64_t run_start, + int64_t run_end) const; + + std::vector groups_; + std::shared_ptr read_schema_; + const int32_t blob_field_idx_; + /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read. + const int32_t row_id_field_idx_; + const int32_t seq_num_field_idx_; + const int32_t read_batch_size_; + std::shared_ptr arrow_pool_; + bool closed_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp new file mode 100644 index 00000000..29dd754c --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/range.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob +/// reader), std::nullopt for null. +using BlobRows = std::vector>; + +class BlobFallbackBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + read_schema_ = arrow::schema(struct_type_->fields()); + } + + static std::string Sentinel() { + return std::string(BlobDefs::PlaceholderSentinelView()); + } + + std::shared_ptr MakeBlobStruct(const BlobRows& rows) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + } + + /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap + /// of n selected rows (with synthetic row ids when the schema does not read them, or the + /// given ranges via GapRanges). + struct SegmentSpec { + std::vector gap_ranges; + std::optional file_rows; + static SegmentSpec Gap(int64_t rows) { + return SegmentSpec{{Range(0, rows - 1)}, std::nullopt}; + } + static SegmentSpec GapRanges(std::vector ranges) { + return SegmentSpec{std::move(ranges), std::nullopt}; + } + static SegmentSpec File(BlobRows rows) { + return SegmentSpec{{}, std::move(rows)}; + } + }; + + std::vector MakeGroup(const std::vector& specs, + int32_t file_batch_size) const { + std::vector segments; + for (const auto& spec : specs) { + if (spec.file_rows) { + auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows), + struct_type_, file_batch_size); + segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}}); + } else { + segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges}); + } + } + return segments; + } + + /// Runs the fallback over the groups with several batch sizes and compares to expected rows. + void CheckFallback(const std::vector>& group_specs, + const BlobRows& expected_rows) const { + auto expected_array = MakeBlobStruct(expected_rows); + for (auto batch_size : arrow::internal::Iota(1, 8)) { + for (auto file_batch_size : {1, 3, 1024}) { + std::vector> groups; + groups.reserve(group_specs.size()); + for (const auto& specs : group_specs) { + groups.push_back(MakeGroup(specs, file_batch_size)); + } + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + + protected: + std::shared_ptr pool_; + std::shared_ptr struct_type_; + std::shared_ptr read_schema_; +}; + +TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) { + // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer + CheckFallback( + {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "u1", "b2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) { + // the newer layer only covers rows 2-3; the gaps stand for placeholders + CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "b1", "u2", "b3"}); + // trailing gap + CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "u1", "b2", "b3"}); + // middle gap between two files of one layer + CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"u0", "b1", "b2", "u3"}); + // a gap segment covering multiple disjoint selected ranges + CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b2", "b3"})}}, + {"b0", "b2", "u3"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) { + // a row that is a placeholder in every layer degrades to null + CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}}, + {"b0", std::nullopt}); + CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}}, + {std::nullopt, std::nullopt}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) { + // a real null in a newer layer wins: null means "updated to null", not "not updated" + CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}}, + {std::nullopt, "u1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestSentinelPrefixedValueIsNotPlaceholder) { + // placeholders are identified by exact equality with the sentinel bytes only: a real value + // that merely starts with them passes through unchanged, whether it falls back or wins as + // the newest layer + std::string prefixed = Sentinel() + "suffix"; + CheckFallback({{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({prefixed, "b1"})}}, + {prefixed, "u1"}); + CheckFallback({{SegmentSpec::File({prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}}, + {prefixed, "b1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) { + CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})}, + {SegmentSpec::File({"PH", "m1", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "m1", "u2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) { + // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9] + CheckFallback( + {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1), + SegmentSpec::File({"u69"})}, + {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4), + SegmentSpec::File({"u48", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}}, + {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) { + // Row-tracking projections: resolved rows keep their layer's row id and sequence number; + // an all-placeholder row keeps its row id (here provided by the newest group's gap + // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the + // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and + // {blob, _SEQUENCE_NUMBER}. + struct RowSpec { + std::optional blob; + int64_t row_id; + int64_t seq_num; + }; + for (bool with_row_id : {true, false}) { + for (bool with_seq_num : {true, false}) { + if (!with_row_id && !with_seq_num) { + continue; + } + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)}; + if (with_row_id) { + fields.push_back(SpecialFields::RowId().field_); + } + if (with_seq_num) { + fields.push_back(SpecialFields::SequenceNumber().field_); + } + auto struct_type = arrow::struct_(fields); + auto schema = arrow::schema(fields); + + auto make_rows = [&](const std::vector& rows) { + std::vector> field_builders = { + std::make_shared()}; + for (size_t i = 1; i < fields.size(); i++) { + field_builders.push_back(std::make_shared()); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row.blob) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row.blob == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok()); + } + int32_t next_field = 1; + if (with_row_id) { + auto builder = static_cast( + struct_builder.field_builder(next_field++)); + EXPECT_TRUE(builder->Append(row.row_id).ok()); + } + if (with_seq_num) { + auto builder = static_cast( + struct_builder.field_builder(next_field)); + EXPECT_TRUE(builder->Append(row.seq_num).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + }; + + for (auto batch_size : arrow::internal::Iota(1, 5)) { + for (auto file_batch_size : {1, 1024}) { + // newest layer (seq 20) covers only row 2; rows 0-1 are a gap + std::vector newest; + newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}}); + newest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique(make_rows({{"u2", 2, 20}}), + struct_type, file_batch_size), + {}}); + // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too + std::vector oldest; + oldest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique( + make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type, + file_batch_size), + {}}); + std::vector> groups; + groups.push_back(std::move(newest)); + groups.push_back(std::move(oldest)); + + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFallbackBatchReader::Create(std::move(groups), schema, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + + // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id + // kept, seq -1), row 2 takes seq 20 + auto expected_array = + make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}}); + auto expected_chunk_array = + std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num + << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + } +} + +TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) { + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024)); + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows"); +} + +TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) { + // a single group needs no fallback + std::vector> single_group; + single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_), + "at least two sequence groups"); + + // the read schema must contain a blob field + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024)); + auto plain_schema = + arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)}); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_), + "should contain a blob field"); + + // groups must not be empty + std::vector> with_empty_group; + with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_group.emplace_back(); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_), + "should not be empty"); + + // a gap segment must cover at least one selected row id + std::vector> with_empty_gap; + with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_), + "at least one selected row id"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp new file mode 100644 index 00000000..a2714a31 --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_view_resolving_batch_reader.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_base.h" +#include "arrow/array/array_binary.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/bytes.h" +#include "paimon/status.h" + +namespace paimon { +BlobViewResolvingBatchReader::BlobViewResolvingBatchReader( + std::unique_ptr&& reader, std::vector read_blob_view_fields, + BlobViewResolver resolver, const std::shared_ptr& pool) + : pool_(pool), + arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + read_blob_view_fields_(std::make_move_iterator(read_blob_view_fields.begin()), + std::make_move_iterator(read_blob_view_fields.end())), + resolver_(std::move(resolver)) {} + +Result BlobViewResolvingBatchReader::NextBatch() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return batch; + } + if (read_blob_view_fields_.empty()) { + return batch; + } + + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (struct_array == nullptr) { + return Status::Invalid( + "invalid batch, BlobViewResolvingBatchReader expects a StructArray batch."); + } + const auto struct_type = struct_array->struct_type(); + + arrow::ArrayVector new_fields = struct_array->fields(); + std::vector field_names; + field_names.reserve(struct_type->num_fields()); + + for (int32_t field_idx = 0; field_idx < struct_type->num_fields(); ++field_idx) { + const auto& field = struct_type->field(field_idx); + field_names.push_back(field->name()); + if (read_blob_view_fields_.find(field->name()) == read_blob_view_fields_.end()) { + continue; + } + const auto& column = struct_array->field(field_idx); + if (auto large_binary_array = std::dynamic_pointer_cast(column)) { + PAIMON_ASSIGN_OR_RAISE(new_fields[field_idx], ResolveBinaryColumn(large_binary_array)); + } else { + return Status::Invalid(fmt::format( + "BlobViewResolvingBatchReader expects blob-view column {} to be LargeBinaryArray.", + field->name())); + } + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr resolved_struct_array, + arrow::StructArray::Make(new_fields, field_names)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*resolved_struct_array, c_array.get(), c_schema.get())); + return batch; +} + +Result> BlobViewResolvingBatchReader::ResolveBinaryColumn( + const std::shared_ptr& blob_view_struct_array) { + arrow::LargeBinaryBuilder builder(arrow_pool_.get()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(blob_view_struct_array->length())); + for (int64_t row = 0; row < blob_view_struct_array->length(); ++row) { + if (blob_view_struct_array->IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + auto view = blob_view_struct_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE(bool is_view_struct, + BlobViewStruct::IsBlobViewStruct(view.data(), view.size())); + if (!is_view_struct) { + return Status::Invalid( + "BlobViewResolvingBatchReader expects a serialized BlobViewStruct in the " + "blob-view column."); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr view_struct, + BlobViewStruct::Deserialize(view.data(), view.size())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr descriptor_bytes, resolver_(*view_struct)); + if (descriptor_bytes == nullptr) { + // null in source table + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append( + reinterpret_cast(descriptor_bytes->data()), descriptor_bytes->size())); + } + std::shared_ptr blob_descriptor_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&blob_descriptor_array)); + return blob_descriptor_array; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader.h b/src/paimon/common/reader/blob_view_resolving_batch_reader.h new file mode 100644 index 00000000..e3b14abb --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" + +namespace paimon { +class BlobViewResolvingBatchReader : public BatchReader { + public: + BlobViewResolvingBatchReader(std::unique_ptr&& reader, + std::vector read_blob_view_fields, + BlobViewResolver resolver, + const std::shared_ptr& pool); + + Result NextBatch() override; + + void Close() override { + reader_->Close(); + } + + std::shared_ptr GetReaderMetrics() const override { + return reader_->GetReaderMetrics(); + } + + private: + Result> ResolveBinaryColumn( + const std::shared_ptr& blob_view_struct_array); + + private: + std::shared_ptr pool_; + std::unique_ptr arrow_pool_; + std::unique_ptr reader_; + std::set read_blob_view_fields_; + BlobViewResolver resolver_; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp new file mode 100644 index 00000000..4091718f --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_view_resolving_batch_reader.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_binary.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class BlobViewResolvingBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + void TearDown() override { + pool_.reset(); + } + + class InMemoryBatchReader : public BatchReader { + public: + explicit InMemoryBatchReader(const std::shared_ptr& struct_array) + : struct_array_(struct_array) { + if (!struct_array_) { + exhausted_ = true; + } + } + + Result NextBatch() override { + if (exhausted_) { + return MakeEofBatch(); + } + exhausted_ = true; + auto c_array = std::make_unique(); + auto c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*struct_array_, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); + } + + std::shared_ptr GetReaderMetrics() const override { + return std::make_shared(); + } + + void Close() override {} + + private: + std::shared_ptr struct_array_; + bool exhausted_ = false; + }; + + std::string MakeBlobViewStructBytes(const std::string& database, const std::string& table, + int32_t field_id, int64_t row_id) const { + Identifier identifier(database, table); + BlobViewStruct view_struct(identifier, field_id, row_id); + auto bytes = view_struct.Serialize(pool_); + return std::string(bytes->data(), bytes->size()); + } + + Result MakeBlobDescriptorBytes(const std::string& uri, int64_t offset, + int64_t length) const { + PAIMON_ASSIGN_OR_RAISE(auto descriptor, BlobDescriptor::Create(uri, offset, length)); + auto bytes = descriptor->Serialize(pool_); + return std::string(bytes->data(), bytes->size()); + } + + std::shared_ptr BuildStructArray(const std::vector& values, + const std::vector& valid) const { + arrow::LargeBinaryBuilder builder; + EXPECT_TRUE(builder.Reserve(static_cast(values.size())).ok()); + for (size_t i = 0; i < values.size(); ++i) { + if (!valid[i]) { + EXPECT_TRUE(builder.AppendNull().ok()); + } else { + EXPECT_TRUE(builder + .Append(reinterpret_cast(values[i].data()), + static_cast(values[i].size())) + .ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", /*nullable=*/true)}; + arrow::ArrayVector arrays = {array}; + auto result = arrow::StructArray::Make(arrays, fields).ValueOrDie(); + return result; + } + + private: + std::shared_ptr pool_; +}; + +TEST_F(BlobViewResolvingBatchReaderTest, TestEofBatch) { + auto inner_reader = std::make_unique(nullptr); + auto resolver = BlobViewResolver([](const BlobViewStruct&) -> Result> { + return std::shared_ptr(); + }); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_OK_AND_ASSIGN(auto batch, reader.NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestEmptyReadBlobViewFields) { + std::string view_bytes = MakeBlobViewStructBytes("db", "table", /*field_id=*/1, /*row_id=*/7); + std::shared_ptr struct_array = BuildStructArray({view_bytes}, {true}); + + bool resolver_called = false; + auto resolver = BlobViewResolver( + [&resolver_called](const BlobViewStruct&) -> Result> { + resolver_called = true; + return std::shared_ptr(); + }); + + auto inner_reader = std::make_unique(struct_array); + BlobViewResolvingBatchReader reader(std::move(inner_reader), /*read_blob_view_fields=*/{}, + std::move(resolver), pool_); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(&reader)); + auto expected_array = std::make_shared(struct_array); + ASSERT_TRUE(expected_array->Equals(*result_array)); + ASSERT_FALSE(resolver_called); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestResolvesBlobViewColumn) { + auto row0_view = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/3, /*row_id=*/100); + auto row1_view = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/3, /*row_id=*/200); + std::shared_ptr src_struct = + BuildStructArray({row0_view, row1_view}, {true, true}); + + ASSERT_OK_AND_ASSIGN(auto expected_row0_descriptor, + MakeBlobDescriptorBytes("/path/a", /*offset=*/0, /*length=*/8)); + ASSERT_OK_AND_ASSIGN(auto expected_row1_descriptor, + MakeBlobDescriptorBytes("/path/b", /*offset=*/16, /*length=*/32)); + + auto resolver = + BlobViewResolver([&](const BlobViewStruct& view_struct) -> Result> { + if (view_struct.RowId() == 100) { + return std::make_shared(expected_row0_descriptor, pool_.get()); + } + if (view_struct.RowId() == 200) { + return std::make_shared(expected_row1_descriptor, pool_.get()); + } + return Status::Invalid("unexpected view struct"); + }); + + auto inner_reader = std::make_unique(src_struct); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(&reader)); + auto struct_array = std::dynamic_pointer_cast(result_array->chunk(0)); + + auto result_blob_column = + std::dynamic_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(result_blob_column->IsNull(0)); + ASSERT_FALSE(result_blob_column->IsNull(1)); + ASSERT_EQ(result_blob_column->GetString(0), expected_row0_descriptor); + ASSERT_EQ(result_blob_column->GetString(1), expected_row1_descriptor); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestResolverError) { + auto view_bytes = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/1, /*row_id=*/5); + std::shared_ptr src_struct = BuildStructArray({view_bytes}, {true}); + auto resolver = BlobViewResolver([](const BlobViewStruct&) -> Result> { + return Status::Invalid("cache miss"); + }); + auto inner_reader = std::make_unique(src_struct); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), "cache miss"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index 8e8fbbce..296a56c6 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -19,6 +19,8 @@ #include "paimon/common/reader/data_evolution_file_reader.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/util.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "fmt/format.h" @@ -26,7 +28,9 @@ #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" + namespace paimon { + Result> DataEvolutionFileReader::Create( std::vector>&& readers, const std::shared_ptr& read_schema, int32_t read_batch_size, @@ -40,8 +44,13 @@ Result> DataEvolutionFileReader::Create return Status::Invalid( "read schema, row offsets and field offsets must have the same size"); } - if (readers.size() <= 1) { - return Status::Invalid("readers size is supposed to be more than 1"); + if (readers.empty()) { + return Status::Invalid("readers must not be empty"); + } + for (int32_t reader_offset : reader_offsets) { + if (reader_offset >= static_cast(readers.size())) { + return Status::Invalid("reader offset is out of range of readers"); + } } return std::unique_ptr( new DataEvolutionFileReader(std::move(readers), read_schema, read_batch_size, @@ -84,6 +93,7 @@ Result DataEvolutionFileReader::NextBatchWithB } const auto& sub_array = array_for_each_reader[reader_offsets_[i]]; assert(sub_array->num_fields() > field_offsets_[i]); + // Each file is already aligned to its read schema by its FieldMappingReader. target_sub_array_vec.push_back(sub_array->field(field_offsets_[i])); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( @@ -168,11 +178,10 @@ Result> DataEvolutionFileReader::NextBatchForSingl if (concat_array_vec.empty()) { return std::shared_ptr(); } - if (concat_array_vec.size() == 1) { - // avoid data copy + if (concat_array_vec.size() == 1 && concat_array_vec[0]->offset() == 0) { + // Avoid data copy when the array is already normalized. return concat_array_vec[0]; } - // TODO(xinyu.lxy) remove data copy for efficiency PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, arrow::Concatenate(concat_array_vec, arrow_pool_.get())); assert(concat_array->length() == total_array_length); diff --git a/src/paimon/common/reader/data_evolution_file_reader.h b/src/paimon/common/reader/data_evolution_file_reader.h index e1bea66e..76494358 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.h +++ b/src/paimon/common/reader/data_evolution_file_reader.h @@ -29,10 +29,12 @@ #include "paimon/result.h" namespace paimon { -/// This is a union reader which contains multiple inner readers. +/// This is a union reader which contains one or more inner readers. /// /// This reader, assembling multiple reader into one big and great reader. The row it produces -/// also come from the readers it contains. +/// also come from the readers it contains. A single inner reader is a valid degenerate case +/// (e.g. a blob-only write): the union still maps its fields into the read schema order and +/// null fills unmatched read fields. /// /// For example, the expected schema for this reader is : int, int, string, int, string, int.(Total /// 6 fields) It contains three inner readers, we call them reader0, reader1 and reader2. diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 39afb15c..9e545c54 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -33,8 +33,8 @@ #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" -namespace paimon::test { +namespace paimon::test { class DataEvolutionFileReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: @@ -120,6 +120,7 @@ class DataEvolutionFileReaderTest : public ::testing::Test, if (result_array == nullptr) { break; } + ASSERT_EQ(result_array->offset(), 0); result_array_vec.push_back(result_array); } ASSERT_EQ(result_array_vec.size(), @@ -165,6 +166,20 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { reader_offsets, field_offsets, pool_), "read schema, row offsets and field offsets must have the same size"); } + { + arrow::FieldVector read_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::int32()), + }; + auto read_schema = arrow::schema(read_fields); + std::vector reader_offsets = {0, 0, 1, 1}; + std::vector field_offsets = {0, 1, 1, 0}; + ASSERT_NOK_WITH_MSG(DataEvolutionFileReader::Create({}, read_schema, /*read_batch_size=*/10, + reader_offsets, field_offsets, pool_), + "readers must not be empty"); + } { std::vector> readers; readers.push_back(nullptr); @@ -181,7 +196,7 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) { ASSERT_NOK_WITH_MSG( DataEvolutionFileReader::Create(std::move(readers), read_schema, /*read_batch_size=*/10, reader_offsets, field_offsets, pool_), - "readers size is supposed to be more than 1"); + "reader offset is out of range of readers"); } } diff --git a/src/paimon/common/reader/delegating_prefetch_reader.h b/src/paimon/common/reader/delegating_prefetch_reader.h index 6d2b09c9..3cbcb08b 100644 --- a/src/paimon/common/reader/delegating_prefetch_reader.h +++ b/src/paimon/common/reader/delegating_prefetch_reader.h @@ -57,8 +57,8 @@ class DelegatingPrefetchReader : public FileBatchReader { return prefetch_reader_->SetReadSchema(read_schema, predicate, selection_bitmap); } - Result GetPreviousBatchFirstRowNumber() const override { - return GetReader()->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return GetReader()->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 95a11d3e..3e8737a3 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -42,6 +42,19 @@ class Schema; namespace paimon { +namespace { + +std::pair ComputeBatchSliceByReadRange( + const std::vector& global_row_ids, const std::pair& read_range) { + auto begin_it = + std::lower_bound(global_row_ids.begin(), global_row_ids.end(), read_range.first); + auto end_it = std::lower_bound(global_row_ids.begin(), global_row_ids.end(), read_range.second); + return {static_cast(std::distance(global_row_ids.begin(), begin_it)), + static_cast(std::distance(global_row_ids.begin(), end_it))}; +} + +} // namespace + Result> PrefetchFileBatchReaderImpl::Create( const std::string& data_file_path, const ReaderBuilder* reader_builder, const std::shared_ptr& fs, uint32_t prefetch_max_parallel_num, int32_t batch_size, @@ -142,6 +155,7 @@ PrefetchFileBatchReaderImpl::~PrefetchFileBatchReaderImpl() { Status PrefetchFileBatchReaderImpl::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { + PAIMON_RETURN_NOT_OK(CleanUp()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(read_schema)); for (const auto& reader : readers_) { @@ -151,11 +165,15 @@ Status PrefetchFileBatchReaderImpl::SetReadSchema( } selection_bitmap_ = selection_bitmap; predicate_ = predicate; - return RefreshReadRanges(); + return RefreshReadRangesAfterCleanUp(); } Status PrefetchFileBatchReaderImpl::RefreshReadRanges() { PAIMON_RETURN_NOT_OK(CleanUp()); + return RefreshReadRangesAfterCleanUp(); +} + +Status PrefetchFileBatchReaderImpl::RefreshReadRangesAfterCleanUp() { bool need_prefetch; PAIMON_ASSIGN_OR_RAISE(auto read_ranges, readers_[0]->GenReadRanges(&need_prefetch)); @@ -171,8 +189,6 @@ Status PrefetchFileBatchReaderImpl::RefreshReadRanges() { need_prefetch_ = need_prefetch; PAIMON_RETURN_NOT_OK(SetReadRanges(FilterReadRanges(read_ranges, selection_bitmap_))); - read_ranges_freshed_ = true; - return Status::OK(); } @@ -219,6 +235,7 @@ Status PrefetchFileBatchReaderImpl::SetReadRanges( for (auto& read_ranges : read_ranges_in_group_) { read_ranges.push_back(eof_range); } + read_ranges_freshed_ = true; return Status::OK(); } @@ -267,6 +284,8 @@ Status PrefetchFileBatchReaderImpl::CleanUp() { read_ranges_.clear(); read_ranges_in_group_.clear(); + current_batch_global_row_ids_.clear(); + read_ranges_freshed_ = false; clean_prefetch_queue(); for (size_t i = 0; i < readers_pos_.size(); i++) { readers_pos_[i]->store(0); @@ -411,28 +430,44 @@ Status PrefetchFileBatchReaderImpl::EnsureReaderPosition( Status PrefetchFileBatchReaderImpl::HandleReadResult( size_t reader_idx, const std::pair& read_range, ReadBatchWithBitmap&& read_batch_with_bitmap) { - PAIMON_ASSIGN_OR_RAISE(uint64_t first_row_number, - readers_[reader_idx]->GetPreviousBatchFirstRowNumber()); auto& prefetch_queue = prefetch_queues_[reader_idx]; if (!BatchReader::IsEofBatch(read_batch_with_bitmap)) { auto& [read_batch, bitmap] = read_batch_with_bitmap; auto& [c_array, c_schema] = read_batch; + std::vector global_row_ids; + global_row_ids.reserve(c_array->length); + for (int64_t i = 0; i < c_array->length; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t global_row_id, + readers_[reader_idx]->GetPreviousBatchFileRowId(i)); + global_row_ids.push_back(global_row_id); + } + if (global_row_ids.empty()) { + ReaderUtils::ReleaseReadBatch(std::move(read_batch)); + return Status::OK(); + } + auto [slice_begin, slice_end] = ComputeBatchSliceByReadRange(global_row_ids, read_range); + // slice_begin should always be 0, records before read_range.first have been consumed or + // filtered out. + if (slice_begin != 0) { + return Status::Invalid(fmt::format("Slice begin is {}, which is not 0.", slice_begin)); + } - if (first_row_number >= read_range.second) { - // fully out of range, data before first_row_number has been filtered out - readers_pos_[reader_idx]->store(first_row_number); + if (0 == slice_end) { + // fully out of range, data before global_row_ids has been filtered out + readers_pos_[reader_idx]->store(global_row_ids[0]); ReaderUtils::ReleaseReadBatch(std::move(read_batch)); return Status::OK(); - } else if (first_row_number + c_array->length > read_range.second) { + } else if (slice_end < c_array->length) { // partially out of range, data before read_range.second has been effectively consumed readers_pos_[reader_idx]->store(read_range.second); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, arrow::ImportArray(c_array.get(), c_schema.get())); - int32_t target_length = read_range.second - first_row_number; - auto array = src_array->Slice(/*offset=*/0, target_length); + auto array = src_array->Slice(0, slice_end); PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::ExportArray(*array, c_array.get(), c_schema.get())); - bitmap.RemoveRange(target_length, src_array->length()); + bitmap.RemoveRange(slice_end, src_array->length()); + global_row_ids = + std::vector(global_row_ids.begin(), global_row_ids.begin() + slice_end); } else { // all within the range, data before readers_[reader_idx]->GetNextRowToRead() has been // effectively consumed @@ -442,11 +477,12 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult( ReaderUtils::ReleaseReadBatch(std::move(read_batch)); return Status::OK(); } - prefetch_queue->push({read_range, std::move(read_batch_with_bitmap), first_row_number}); + prefetch_queue->push( + {read_range, std::move(read_batch_with_bitmap), std::move(global_row_ids)}); } else { std::pair eof_range; PAIMON_ASSIGN_OR_RAISE(eof_range, EofRange()); - prefetch_queue->push({eof_range, std::move(read_batch_with_bitmap), first_row_number}); + prefetch_queue->push({eof_range, std::move(read_batch_with_bitmap), {}}); readers_pos_[reader_idx]->store(std::numeric_limits::max()); } return Status::OK(); @@ -529,7 +565,7 @@ Result PrefetchFileBatchReaderImpl::NextBatchW std::unique_lock lock(working_mutex_); cv_.notify_one(); } - previous_batch_first_row_num_ = prefetch_batch.value().previous_batch_first_row_num; + current_batch_global_row_ids_ = std::move(prefetch_batch.value().global_row_ids); return std::move(prefetch_batch).value().batch; } } @@ -539,7 +575,7 @@ Result PrefetchFileBatchReaderImpl::NextBatchW assert(false); return Status::Invalid("peek batch not suppose to be nullptr"); } - previous_batch_first_row_num_ = peek_batch->previous_batch_first_row_num; + current_batch_global_row_ids_.clear(); return BatchReader::MakeEofBatchWithBitmap(); } if (value_count == prefetch_queues_.size()) { @@ -573,8 +609,19 @@ Result> PrefetchFileBatchReaderImpl::GetFileSchem return readers_[0]->GetFileSchema(); } -Result PrefetchFileBatchReaderImpl::GetPreviousBatchFirstRowNumber() const { - return previous_batch_first_row_num_; +Result PrefetchFileBatchReaderImpl::GetPreviousBatchFileRowId( + uint64_t batch_row_id) const { + if (current_batch_global_row_ids_.empty()) { + return Status::Invalid( + "Last batch is not read or last batch is empty, cannot get previous batch global row " + "id"); + } + if (batch_row_id >= current_batch_global_row_ids_.size()) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", batch_row_id, + current_batch_global_row_ids_.size())); + } + return current_batch_global_row_ids_[batch_row_id]; } Result PrefetchFileBatchReaderImpl::GetNumberOfRows() const { diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h index f2916da3..36673e8b 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h @@ -78,7 +78,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { const std::optional& selection_bitmap) override; Status SeekToRow(uint64_t row_number) override; - Result GetPreviousBatchFirstRowNumber() const override; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; Result GetNumberOfRows() const override; uint64_t GetNextRowToRead() const override; void Close() override; @@ -107,7 +107,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { struct PrefetchBatch { std::pair read_range; BatchReader::ReadBatchWithBitmap batch; - uint64_t previous_batch_first_row_num; + std::vector global_row_ids; }; PrefetchFileBatchReaderImpl( @@ -131,6 +131,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { static std::vector>> DispatchReadRanges( const std::vector>& read_ranges, size_t reader_count); + Status RefreshReadRangesAfterCleanUp(); Result> EofRange() const; std::optional> GetCurrentReadRange(size_t reader_idx) const; Status EnsureReaderPosition(size_t reader_idx, @@ -162,7 +163,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { std::unique_ptr background_thread_; Status read_status_; std::atomic is_shutdown_ = false; - uint64_t previous_batch_first_row_num_ = std::numeric_limits::max(); + std::vector current_batch_global_row_ids_; bool need_prefetch_ = false; bool read_ranges_freshed_ = false; const uint32_t prefetch_queue_capacity_; diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index d9b58218..30ecdf65 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -18,6 +18,8 @@ */ #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" +#include +#include #include #include "arrow/compute/api.h" @@ -92,7 +94,7 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder { Result> Build( const std::shared_ptr& path) const override { - size_t index = build_count_++; + size_t index = build_count_.fetch_add(1); Status set_read_ranges_status = index < set_read_ranges_statuses_.size() ? set_read_ranges_statuses_[index] : Status::OK(); @@ -107,7 +109,7 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder { std::vector> read_ranges_; bool need_prefetch_ = true; std::vector set_read_ranges_statuses_; - mutable size_t build_count_ = 0; + mutable std::atomic build_count_{0}; }; struct TestParam { @@ -124,7 +126,7 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, data_type_ = arrow::struct_(fields_); mock_fs_ = std::make_shared(); local_fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); dir_ = ::paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); } @@ -195,14 +197,16 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, EXPECT_OK_AND_ASSIGN(std::unique_ptr file_format, FileFormatFactory::Get(file_format_str, {})); EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(batch_size)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(prefetch_max_parallel_num - 1)); EXPECT_OK_AND_ASSIGN( std::unique_ptr reader, PrefetchFileBatchReaderImpl::Create( PathUtil::JoinPath(dir_->Str(), "file." + file_format->Identifier()), reader_builder.get(), local_fs_, prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, - CreateDefaultExecutor(prefetch_max_parallel_num - 1), - /*initialize_read_ranges=*/false, cache_mode, CacheConfig(), GetDefaultPool())); + executor, /*initialize_read_ranges=*/false, cache_mode, CacheConfig(), + GetDefaultPool())); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); EXPECT_TRUE(arrow_status.ok()); @@ -245,6 +249,28 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, std::shared_ptr executor_; }; +static Result, std::vector>> +CollectResultAndRowIds(FileBatchReader* reader) { + arrow::ArrayVector result_array_vector; + std::vector row_ids; + while (true) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader)); + if (batch == nullptr) { + break; + } + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, + reader->GetPreviousBatchFileRowId(batch->chunk(0)->length() - 1)); + row_ids.push_back(file_row_id); + result_array_vector.push_back(batch->chunk(0)); + } + if (result_array_vector.empty()) { + return std::make_pair(std::shared_ptr(), row_ids); + } + auto result_array = std::make_shared(result_array_vector); + return std::make_pair(result_array, row_ids); +} + std::vector PrepareTestParam() { std::vector values = { TestParam{"parquet", PrefetchCacheMode::ALWAYS}, @@ -278,13 +304,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestSimple) { /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 101); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 100); } } @@ -485,7 +511,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRangesReturnErrorWhenPushDownFail data_array, data_type_, batch_size, /*read_ranges=*/{{0, 50}, {50, 100}}, /*need_prefetch=*/true, - /*set_read_ranges_statuses=*/{Status::OK(), Status::IOError("set read ranges failed")}); + /*set_read_ranges_statuses=*/ + {Status::IOError("set read ranges failed"), Status::IOError("set read ranges failed")}); ASSERT_OK_AND_ASSIGN( auto reader, @@ -529,7 +556,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size); CacheConfig invalid_cache_config( /*buffer_size_limit=*/512 * 1024, - /*range_size_limit=*/static_cast(std::numeric_limits::max()) + 1, + /*range_size_limit=*/4 * 1024, /*hole_size_limit=*/8 * 1024, /*pre_buffer_limit=*/128 * 1024); @@ -545,9 +572,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed auto prefetch_reader = dynamic_cast(reader.get()); prefetch_reader->Workloop(); - Status status = prefetch_reader->GetReadStatus(); - ASSERT_FALSE(status.ok()); - ASSERT_TRUE(status.IsInvalid()); + ASSERT_NOK_WITH_MSG(prefetch_reader->GetReadStatus(), + "range size limit 4096 should be larger than hole size limit 8192"); } TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenShutdown) { @@ -600,13 +626,12 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLargeBatchSize) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 101); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 100); auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); } TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) { @@ -628,11 +653,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) { } arrow::ArrayVector result_array_vector; - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap()); auto& [batch, bitmap] = batch_with_bitmap; ASSERT_EQ(batch.first->length, bitmap.Cardinality()); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(reader->GetPreviousBatchFileRowId(0).value(), 0); ASSERT_OK_AND_ASSIGN(auto array, ReadResultCollector::GetArray(std::move(batch))); result_array_vector.push_back(array); ASSERT_OK(prefetch_reader->GetReadStatus()); @@ -673,9 +698,9 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { ->SetNextBatchStatus(Status::IOError("mock error")); } - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); auto batch_result = reader->NextBatchWithBitmap(); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_FALSE(batch_result.ok()); ASSERT_TRUE(batch_result.status().IsIOError()); ASSERT_FALSE(prefetch_reader->is_shutdown_); @@ -684,7 +709,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { // call NextBatch again, will still return error status auto batch_result2 = reader->NextBatchWithBitmap(); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_FALSE(batch_result2.ok()); ASSERT_TRUE(batch_result2.status().IsIOError()); } @@ -701,12 +726,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithEmptyData) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 0); - ASSERT_FALSE(result_array); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids.size(), 0); + ASSERT_FALSE(array_and_row_ids.first); } TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) { @@ -721,16 +745,16 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 10); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 9); auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); // continue to call NextBatch() after reading eof ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap)); } @@ -813,6 +837,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) { TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCompleteFiltering) { auto [file_format, cache_mode] = GetParam(); auto data_array = PrepareArray(90); + int32_t batch_size = 10; PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/30); auto schema = arrow::schema(fields_); ASSERT_OK_AND_ASSIGN(auto predicate, @@ -823,21 +848,19 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom FieldType::BIGINT, Literal(70l)), })); - auto reader = - PreparePrefetchReader(file_format, schema.get(), predicate, - /*selection_bitmap=*/std::nullopt, - /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 90); + auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3, + cache_mode); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); arrow::ArrayVector expected_array_vector; + std::vector expected_row_ids = {9, 19, 29, 69, 79, 89}; expected_array_vector.push_back(data_array->Slice(0, 30)); expected_array_vector.push_back(data_array->Slice(60, 30)); auto expected_array = std::make_shared(expected_array_vector); - ASSERT_TRUE(CheckEqual(expected_array, result_array)); + ASSERT_TRUE(expected_array->Equals(array_and_row_ids.first)); + ASSERT_EQ(expected_row_ids, array_and_row_ids.second); } /// There are three stripes: [0,30), [30,60), [60,90). Each stripe has 3 row groups. @@ -847,6 +870,7 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithOrcPredicatePushdownWithRowGroupGranularity) { auto [file_format, cache_mode] = GetParam(); auto data_array = PrepareArray(90); + int32_t batch_size = 10; PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10); auto schema = arrow::schema(fields_); @@ -858,22 +882,21 @@ TEST_P(PrefetchFileBatchReaderImplTest, FieldType::BIGINT, Literal(70l)), })); - auto reader = - PreparePrefetchReader(file_format, schema.get(), predicate, - /*selection_bitmap=*/std::nullopt, - /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); + auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3, + cache_mode); ASSERT_OK(reader->RefreshReadRanges()); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 90); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); arrow::ArrayVector expected_array_vector; + std::vector expected_row_ids = {9, 19, 79, 89}; expected_array_vector.push_back(data_array->Slice(0, 20)); expected_array_vector.push_back(data_array->Slice(70, 20)); auto expected_array = std::make_shared(expected_array_vector); - ASSERT_TRUE(CheckEqual(expected_array, result_array)); + ASSERT_TRUE(expected_array->Equals(array_and_row_ids.first)); + ASSERT_EQ(expected_row_ids, array_and_row_ids.second); } TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) { @@ -907,4 +930,55 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) { ASSERT_TRUE(result_chunk_array->Equals(expected_chunk_array)); } +TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) { + auto [file_format, cache_mode] = GetParam(); + auto data_array = PrepareArray(90); + PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10); + auto schema = arrow::schema(fields_); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({ + PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(20l), Literal(29l)), + PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(70l), Literal(79l)), + })); + + auto reader = + PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 20 + i); + } + + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 70 + i); + } + + // Set read schema again + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + predicate = PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(30l), Literal(49l)); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), predicate, std::nullopt)); + + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 30 + i); + } + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 40 + i); + } +} + } // namespace paimon::test diff --git a/src/paimon/common/sst/block_cache.h b/src/paimon/common/sst/block_cache.h index 4fd8bb2a..8e6a8631 100644 --- a/src/paimon/common/sst/block_cache.h +++ b/src/paimon/common/sst/block_cache.h @@ -23,8 +23,8 @@ #include #include "paimon/common/io/cache/cache_manager.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/fs/file_system.h" +#include "paimon/memory/memory_segment.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/sst/block_handle.h b/src/paimon/common/sst/block_handle.h index c00a5af6..7244b516 100644 --- a/src/paimon/common/sst/block_handle.h +++ b/src/paimon/common/sst/block_handle.h @@ -20,9 +20,9 @@ #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/sst/bloom_filter_handle.h b/src/paimon/common/sst/bloom_filter_handle.h index 9796bcba..0814231b 100644 --- a/src/paimon/common/sst/bloom_filter_handle.h +++ b/src/paimon/common/sst/bloom_filter_handle.h @@ -18,11 +18,13 @@ #pragma once -#include +#include +#include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" +#include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/sst/sst_file_reader.cpp b/src/paimon/common/sst/sst_file_reader.cpp index 9639bcd3..315bc199 100644 --- a/src/paimon/common/sst/sst_file_reader.cpp +++ b/src/paimon/common/sst/sst_file_reader.cpp @@ -67,7 +67,7 @@ Result> SstFileReader::Create( Result> SstFileReader::CreateForSortLookupStore( const std::shared_ptr& in, MemorySlice::SliceComparator comparator, const std::shared_ptr& block_cache, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_len, in->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_len, in->Length()); PAIMON_RETURN_NOT_OK( in->Seek(file_len - SortLookupStoreFooter::ENCODED_LENGTH, SeekOrigin::FS_SEEK_SET)); auto footer_bytes = Bytes::AllocateBytes(SortLookupStoreFooter::ENCODED_LENGTH, pool.get()); diff --git a/src/paimon/common/sst/sst_file_utils.h b/src/paimon/common/sst/sst_file_utils.h index 24c61788..25a9f314 100644 --- a/src/paimon/common/sst/sst_file_utils.h +++ b/src/paimon/common/sst/sst_file_utils.h @@ -17,11 +17,9 @@ */ #pragma once -#include #include "fmt/format.h" #include "paimon/common/compression/block_compression_type.h" -#include "paimon/common/memory/memory_slice.h" namespace paimon { diff --git a/src/paimon/common/sst/sst_file_writer.cpp b/src/paimon/common/sst/sst_file_writer.cpp index 003a386b..ec736e33 100644 --- a/src/paimon/common/sst/sst_file_writer.cpp +++ b/src/paimon/common/sst/sst_file_writer.cpp @@ -18,7 +18,6 @@ #include "paimon/common/sst/sst_file_writer.h" -#include "paimon/common/sst/sst_file_utils.h" #include "paimon/common/utils/crc32c.h" #include "paimon/common/utils/murmurhash_utils.h" diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h index b4319649..74b95b19 100644 --- a/src/paimon/common/table/special_fields.h +++ b/src/paimon/common/table/special_fields.h @@ -24,6 +24,7 @@ #include "arrow/type_fwd.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/utils/special_field_ids.h" namespace paimon { @@ -65,14 +66,15 @@ struct SpecialFields { return data_field; } - static bool IsSpecialFieldName(const std::string& field_name) { - if (field_name == SequenceNumber().Name() || field_name == ValueKind().Name() || - field_name == RowKind().Name() || field_name == RowId().Name() || - field_name == IndexScore().Name()) { + static bool IsSystemField(const std::string& field_name) { + if (StringUtils::StartsWith(field_name, KEY_FIELD_PREFIX)) { return true; } - return false; + return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() || + field_name == RowKind().Name() || field_name == RowId().Name() || + field_name == IndexScore().Name(); } + // TODO(xinyu.lxy): add a func to complete row-tracking fields static std::shared_ptr CompleteSequenceAndValueKindField( diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp index 0c17918c..68e805fd 100644 --- a/src/paimon/common/table/special_fields_test.cpp +++ b/src/paimon/common/table/special_fields_test.cpp @@ -59,13 +59,14 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) { ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2); } -TEST(SpecialFieldsTest, TestIsSpecialFieldName) { - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_SEQUENCE_NUMBER")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_VALUE_KIND")); - ASSERT_FALSE(SpecialFields::IsSpecialFieldName("VALUE_KIND")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("rowkind")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_ROW_ID")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_INDEX_SCORE")); +TEST(SpecialFieldsTest, TestIsSystemField) { + ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER")); + ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND")); + ASSERT_FALSE(SpecialFields::IsSystemField("VALUE_KIND")); + ASSERT_TRUE(SpecialFields::IsSystemField("rowkind")); + ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID")); + ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE")); + ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0")); } } // namespace paimon::test diff --git a/src/paimon/common/types/array_type.h b/src/paimon/common/types/array_type.h index d03c0da9..cecd5bc2 100644 --- a/src/paimon/common/types/array_type.h +++ b/src/paimon/common/types/array_type.h @@ -46,8 +46,10 @@ class ArrayType : public DataType { auto type = arrow::internal::checked_cast(type_.get()); auto value_field = type->value_field(); + // The element metadata is load-bearing: it is what marks an extension type such as + // VARIANT. std::shared_ptr data_type = - DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr); + DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); obj.AddMember(rapidjson::StringRef("element"), RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator); return obj; diff --git a/src/paimon/common/types/data_field.cpp b/src/paimon/common/types/data_field.cpp index 4aa10924..97a167b0 100644 --- a/src/paimon/common/types/data_field.cpp +++ b/src/paimon/common/types/data_field.cpp @@ -176,4 +176,38 @@ Result> DataField::ProjectFields( return projected_fields; } +std::shared_ptr DataField::MergeFieldMetadataByWhitelist( + const std::shared_ptr& target_field, + const std::shared_ptr& source_field, + const std::vector& metadata_keys_whitelist) { + if (!source_field || !source_field->HasMetadata() || !source_field->metadata()) { + return target_field; + } + + std::unordered_map metadata_map; + for (const auto& key : metadata_keys_whitelist) { + auto metadata_value_result = source_field->metadata()->Get(key); + if (metadata_value_result.ok()) { + metadata_map[key] = metadata_value_result.ValueUnsafe(); + } + } + + if (metadata_map.empty()) { + return target_field; + } + + auto metadata = std::make_shared(metadata_map); + return target_field->WithMergedMetadata(metadata); +} + +DataField DataField::MergeFieldMetadataByWhitelist( + const DataField& target_field, const DataField& source_field, + const std::vector& metadata_keys_whitelist) { + return DataField( + target_field.Id(), + MergeFieldMetadataByWhitelist(target_field.ArrowField(), source_field.ArrowField(), + metadata_keys_whitelist), + target_field.Description()); +} + } // namespace paimon diff --git a/src/paimon/common/types/data_field.h b/src/paimon/common/types/data_field.h index ada3ce90..4ca6d59f 100644 --- a/src/paimon/common/types/data_field.h +++ b/src/paimon/common/types/data_field.h @@ -44,6 +44,9 @@ class DataField : public Jsonizable { static constexpr char FIELD_ID[] = "paimon.id"; static constexpr char DESCRIPTION[] = "paimon.description"; + /// Metadata key for map field selected keys. The value is a comma-separated + /// string of key names, e.g. 'key1,key2'. Only string-keyed maps are supported. + static constexpr char MAP_SELECTED_KEYS[] = "paimon.map.selected-keys"; public: static std::shared_ptr ConvertDataFieldToArrowField(const DataField& field); @@ -66,6 +69,17 @@ class DataField : public Jsonizable { const std::vector& fields, const std::optional>& projected_cols); + /// Merge whitelisted metadata from source_field into target_field. + static std::shared_ptr MergeFieldMetadataByWhitelist( + const std::shared_ptr& target_field, + const std::shared_ptr& source_field, + const std::vector& metadata_keys_whitelist); + + /// Merge whitelisted metadata from source_field into target_field and keep target id/desc. + static DataField MergeFieldMetadataByWhitelist( + const DataField& target_field, const DataField& source_field, + const std::vector& metadata_keys_whitelist); + int32_t Id() const { return id_; } diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 7b71f53d..623d4ca2 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -26,6 +26,7 @@ #include "arrow/util/checked_cast.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/array_type.h" #include "paimon/common/types/map_type.h" #include "paimon/common/types/row_type.h" @@ -52,6 +53,11 @@ std::unique_ptr DataType::Create( case arrow::Type::type::LIST: return std::make_unique(type, nullable, metadata); case arrow::Type::type::STRUCT: + if (VariantTypeUtils::IsVariantMetadata(metadata)) { + // A variant field is physically a struct but is a scalar + // VARIANT type in the paimon type system, not a ROW type. + return std::unique_ptr(new DataType(type, nullable, metadata)); + } return std::make_unique(type, nullable, metadata); default: return std::unique_ptr(new DataType(type, nullable, metadata)); @@ -119,16 +125,23 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t arrow::internal::checked_pointer_cast(type); return TimestampToString(timestamp_type); } + case arrow::Type::type::STRUCT: { + if (VariantTypeUtils::IsVariantMetadata(metadata_)) { + return "VARIANT"; + } + break; + } case arrow::Type::type::LARGE_BINARY: { // TODO(xinyu): change binary to large binary? if (BlobUtils::IsBlobMetadata(metadata_)) { return "BLOB"; } - [[fallthrough]]; + break; } default: - throw std::invalid_argument(fmt::format("unknown type {}", type->ToString())); + break; } + throw std::invalid_argument(fmt::format("unknown type {}", type->ToString())); } } // namespace paimon diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index f4ab2315..950308a5 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -82,9 +84,16 @@ struct Token { std::string value; }; +// Extension type attributes of a parsed atomic type. BLOB and VARIANT parse to plain arrow +// types (large_binary / struct) and need field-level metadata markers applied by the caller. +struct AtomicTypeAttributes { + bool is_blob = false; + bool is_variant = false; +}; + // nullptr is returned in the case of parsing failed Result> ParseAtomicType(const std::string& str, bool* nullable, - bool* is_blob); + AtomicTypeAttributes* attributes); std::vector Tokenize(const std::string& chars); bool IsWhitespace(char character); bool IsDelimiter(char character); @@ -138,6 +147,7 @@ enum class Keyword : int32_t { MAP, ROW, BLOB, + VARIANT, // NULL is keyword in c++ NULL_, RAW, @@ -186,6 +196,7 @@ const std::map& Keywords() { {"MAP", Keyword::MAP}, {"ROW", Keyword::ROW}, {"BLOB", Keyword::BLOB}, + {"VARIANT", Keyword::VARIANT}, {"NULL", Keyword::NULL_}, {"RAW", Keyword::RAW}, {"LEGACY", Keyword::LEGACY}, @@ -198,7 +209,8 @@ class TokenParser { TokenParser(const std::string& input_string, const std::vector& tokens) : input_string_(input_string), tokens_(tokens) {} - Result> ParseTokens(bool* nullable, bool* is_blob); + Result> ParseTokens(bool* nullable, + AtomicTypeAttributes* attributes); private: inline const Token& GetToken() const { @@ -227,9 +239,9 @@ class TokenParser { bool HasNextToken(const std::vector& types) const; bool HasNextToken(const std::vector& keywords) const; Result ParseNullability(); - Result> ParseTypeWithNullability(bool* nullable, - bool* is_blob); - Result> ParseTypeByKeyword(bool* is_blob); + Result> ParseTypeWithNullability( + bool* nullable, AtomicTypeAttributes* attributes); + Result> ParseTypeByKeyword(AtomicTypeAttributes* attributes); Result ParseStringLength(); template Result> ParseStringType(); @@ -247,11 +259,11 @@ class TokenParser { }; Result> ParseAtomicType(const std::string& str, bool* nullable, - bool* is_blob) { + AtomicTypeAttributes* attributes) { try { std::vector tokens = Tokenize(str); TokenParser converter(str, tokens); - return converter.ParseTokens(nullable, is_blob); + return converter.ParseTokens(nullable, attributes); } catch (...) { return Status::Invalid("parse atomic type failed."); } @@ -373,9 +385,10 @@ int32_t ConsumeIdentifier(const std::string& chars, int32_t cursor, std::ostring return cursor - 1; } -Result> TokenParser::ParseTokens(bool* nullable, bool* is_blob) { +Result> TokenParser::ParseTokens( + bool* nullable, AtomicTypeAttributes* attributes) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr type, - ParseTypeWithNullability(nullable, is_blob)); + ParseTypeWithNullability(nullable, attributes)); if (HasRemainingTokens()) { PAIMON_RETURN_NOT_OK(NextToken()); return Status::Invalid(fmt::format("Unexpected token: {}", GetToken().value)); @@ -454,9 +467,10 @@ Result TokenParser::ParseNullability() { return true; } -Result> TokenParser::ParseTypeWithNullability(bool* nullable, - bool* is_blob) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_type, ParseTypeByKeyword(is_blob)); +Result> TokenParser::ParseTypeWithNullability( + bool* nullable, AtomicTypeAttributes* attributes) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_type, + ParseTypeByKeyword(attributes)); PAIMON_ASSIGN_OR_RAISE(*nullable, ParseNullability()); // special case: suffix notation for ARRAY types if (HasNextToken({Keyword::ARRAY}) || HasNextToken({Keyword::MULTISET})) { @@ -465,15 +479,26 @@ Result> TokenParser::ParseTypeWithNullability(b return data_type; } -Result> TokenParser::ParseTypeByKeyword(bool* is_blob) { +Result> TokenParser::ParseTypeByKeyword( + AtomicTypeAttributes* attributes) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::KEYWORD)); switch (TokenAsKeyword()) { + case Keyword::CHAR: + case Keyword::VARCHAR: + return ParseStringType(); + case Keyword::BINARY: + case Keyword::VARBINARY: + return ParseStringType(); case Keyword::BYTES: return arrow::binary(); case Keyword::BLOB: { - *is_blob = true; + attributes->is_blob = true; return arrow::large_binary(); } + case Keyword::VARIANT: { + attributes->is_variant = true; + return VariantTypeUtils::UnshreddedStructType(); + } case Keyword::STRING: return arrow::utf8(); case Keyword::BOOLEAN: @@ -511,9 +536,14 @@ Result TokenParser::ParseStringLength() { if (HasNextToken({TokenType::BEGIN_PARAMETER})) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_PARAMETER)); PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT)); - auto length = TokenAsInt(); + int64_t length = std::stoll(GetToken().value); + if (length < 1 || length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("length must be between 1 and {} (both inclusive), but was {}", + std::numeric_limits::max(), length)); + } PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_PARAMETER)); - return length; + return static_cast(length); } // implicit length return -1; @@ -603,11 +633,13 @@ Result> DataTypeJsonParser::ParseType( Result> DataTypeJsonParser::ParseAtomicTypeField( const std::string& name, const rapidjson::Value& type_json_value) { bool nullable = true; - bool is_blob = false; + AtomicTypeAttributes attributes; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr type, - ParseAtomicType(type_json_value.GetString(), &nullable, &is_blob)); - if (is_blob) { + ParseAtomicType(type_json_value.GetString(), &nullable, &attributes)); + if (attributes.is_blob) { return BlobUtils::ToArrowField(name, nullable); + } else if (attributes.is_variant) { + return VariantTypeUtils::ToArrowField(name, nullable); } else { return arrow::field(name, type, nullable); } diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index 25b2ef59..bbb4fb52 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -23,6 +23,7 @@ #include #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -114,6 +115,16 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { {"TIMESTAMP_LTZ(9)", arrow::timestamp(arrow::TimeUnit::NANO, timezone)}, {"BYTES", arrow::binary()}, {"STRING", arrow::utf8()}, + {"CHAR", arrow::utf8()}, + {"CHAR(10)", arrow::utf8()}, + {"VARCHAR", arrow::utf8()}, + {"VARCHAR(10)", arrow::utf8()}, + {"BINARY", arrow::binary()}, + {"BINARY(10)", arrow::binary()}, + {"VARBINARY", arrow::binary()}, + {"VARBINARY(10)", arrow::binary()}, + {"CHAR(1)", arrow::utf8()}, + {"VARCHAR(2147483647)", arrow::utf8()}, }; for (const auto& test_case : test_cases) { @@ -128,12 +139,29 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { ASSERT_TRUE(field->type()->Equals(test_case.second)); } + // VARIANT parses to a variant-marked struct field. + for (const char* variant_str : {"VARIANT", "VARIANT NOT NULL"}) { + rapidjson::Document doc; + rapidjson::Value value(variant_str, doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + DataTypeJsonParser::ParseType("variant_field", value)); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_EQ(field->nullable(), std::string(variant_str) == "VARIANT"); + ASSERT_TRUE(field->type()->Equals(VariantTypeUtils::UnshreddedStructType())); + } + // Invalid case { rapidjson::Document invalid_doc; rapidjson::Value value("VARCHAR(test)", invalid_doc.GetAllocator()); ASSERT_NOK(DataTypeJsonParser::ParseType("field_name", value)); } + for (const char* invalid_type : {"VARCHAR(0)", "VARBINARY(0)", "VARCHAR(2147483648)"}) { + rapidjson::Document invalid_doc; + rapidjson::Value value(invalid_type, invalid_doc.GetAllocator()); + ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), + "length must be between 1 and 2147483647"); + } { rapidjson::Document invalid_doc; rapidjson::Value value("TIMESTAMP(4)", invalid_doc.GetAllocator()); diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index 137cf854..9568c3ff 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -24,7 +24,10 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/rapidjson_util.h" namespace paimon::test { @@ -65,6 +68,19 @@ TEST(DataTypeTest, DataTypeToString) { DataType blob_type(blob_field->type(), blob_field->nullable(), blob_field->metadata()); ASSERT_EQ(blob_type.DataTypeToString(blob_field->type()), "BLOB"); } + { + std::shared_ptr variant_field = VariantTypeUtils::ToArrowField("f3_variant"); + DataType variant_type(variant_field->type(), variant_field->nullable(), + variant_field->metadata()); + ASSERT_EQ(variant_type.DataTypeToString(variant_field->type()), "VARIANT"); + // A variant field is a scalar VARIANT type, not a ROW type. + auto created = DataType::Create(variant_field->type(), variant_field->nullable(), + variant_field->metadata()); + rapidjson::Document doc; + auto json_value = created->ToJson(&doc.GetAllocator()); + ASSERT_TRUE(json_value.IsString()); + ASSERT_EQ(std::string(json_value.GetString()), "VARIANT"); + } ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::date32()), "DATE"); auto decimal_type1 = arrow::decimal128(10, 2); @@ -98,4 +114,38 @@ TEST(DataTypeTest, DataTypeToString) { ASSERT_THROW(dummy_data_type.DataTypeToString(unknown_type), std::invalid_argument); } +TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) { + // ARRAY and MAP must carry their children's metadata into the serialized type, because that + // is what marks an extension type. Dropping it serialized a VARIANT as its physical + // `struct` ROW, whose fixed child ids 0/1 then broke reading the schema back. + auto to_json = [](const std::shared_ptr& field) { + auto data_type = DataType::Create(field->type(), field->nullable(), field->metadata()); + rapidjson::Document doc; + auto value = data_type->ToJson(&doc.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return std::string(buffer.GetString()); + }; + + auto variant_field = VariantTypeUtils::ToArrowField("element"); + auto array_field = arrow::field("arr", arrow::list(variant_field)); + ASSERT_EQ(to_json(array_field), R"({"type":"ARRAY","element":"VARIANT"})"); + + auto map_field = + arrow::field("m", arrow::map(arrow::utf8(), VariantTypeUtils::ToArrowField("value"))); + ASSERT_EQ(to_json(map_field), R"({"type":"MAP","key":"STRING NOT NULL","value":"VARIANT"})"); + + // BLOB is marked the same way and was degraded to its physical BYTES the same way. + auto blob_array = arrow::field("b", arrow::list(BlobUtils::ToArrowField("element", true))); + ASSERT_EQ(to_json(blob_array), R"({"type":"ARRAY","element":"BLOB"})"); + + // Child metadata only ever selects an extension type, so ordinary elements are unaffected by + // it being carried over. + auto plain_child = arrow::field("element", arrow::int32(), /*nullable=*/true, + arrow::KeyValueMetadata::Make({DataField::FIELD_ID}, {"7"})); + ASSERT_EQ(to_json(arrow::field("a", arrow::list(plain_child))), + R"({"type":"ARRAY","element":"INT"})"); +} + } // namespace paimon::test diff --git a/src/paimon/common/types/map_type.h b/src/paimon/common/types/map_type.h index 7749dfca..29fd8bbd 100644 --- a/src/paimon/common/types/map_type.h +++ b/src/paimon/common/types/map_type.h @@ -52,13 +52,15 @@ class MapType : public DataType { *allocator); auto type = arrow::internal::checked_cast(type_.get()); auto key_field = type->key_field(); + // The key and value metadata is load-bearing: it is what marks an extension type such as + // VARIANT. std::shared_ptr key_data_type = - DataType::Create(key_field->type(), key_field->nullable(), /*metadata=*/nullptr); + DataType::Create(key_field->type(), key_field->nullable(), key_field->metadata()); obj.AddMember(rapidjson::StringRef("key"), RapidJsonUtil::SerializeValue(*key_data_type, allocator).Move(), *allocator); auto value_field = type->item_field(); std::shared_ptr value_data_type = - DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr); + DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); obj.AddMember(rapidjson::StringRef("value"), RapidJsonUtil::SerializeValue(*value_data_type, allocator).Move(), *allocator); diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp index 941c18dd..01429df6 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp @@ -20,13 +20,12 @@ #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include +#include #include #include "arrow/api.h" -#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/math.h" -#include "paimon/common/utils/options_utils.h" #include "paimon/fs/file_system.h" #include "paimon/macros.h" #include "paimon/result.h" @@ -36,11 +35,10 @@ namespace paimon { namespace { -template -arrow::Status ValidateArrowIoRange(From value, const char* name) { - if (!InRange(value)) { - return arrow::Status::Invalid(fmt::format("{} value {} is out of bound of type {}", name, - value, OptionsUtils::GetTypeName())); +arrow::Status ValidateArrowIoRange(int64_t value, const char* name) { + Status status = ValidateValueNonNegative(value, name); + if (!status.ok()) { + return ToArrowStatus(status); } return arrow::Status::OK(); } @@ -48,9 +46,14 @@ arrow::Status ValidateArrowIoRange(From value, const char* name) { } // namespace ArrowInputStreamAdapter::ArrowInputStreamAdapter( - const std::shared_ptr& input_stream, - const std::shared_ptr& pool, uint64_t file_size) - : input_stream_(input_stream), pool_(pool), file_size_(file_size) {} + const std::shared_ptr& input_stream, int64_t file_size, + const std::shared_ptr& pool) + : input_stream_(input_stream), + pool_(pool), + file_size_(file_size), + storage_read_bytes_(std::make_shared>(0)) { + assert(file_size >= 0); +} ArrowInputStreamAdapter::~ArrowInputStreamAdapter() { [[maybe_unused]] auto status = DoClose(); @@ -61,12 +64,14 @@ arrow::Status ArrowInputStreamAdapter::Seek(int64_t position) { } arrow::Result ArrowInputStreamAdapter::Read(int64_t nbytes, void* out) { - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); - Result read_bytes = - input_stream_->Read(static_cast(out), static_cast(nbytes)); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); + Result read_bytes = input_stream_->Read(static_cast(out), nbytes); if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } + if (storage_read_bytes_) { + storage_read_bytes_->fetch_add(static_cast(read_bytes.value())); + } return read_bytes.value(); } @@ -82,13 +87,15 @@ arrow::Result> ArrowInputStreamAdapter::Read(int6 arrow::Result ArrowInputStreamAdapter::ReadAt(int64_t position, int64_t nbytes, void* out) { - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(position, "position")); - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); - Result read_bytes = input_stream_->Read( - static_cast(out), static_cast(nbytes), static_cast(position)); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(position, "position")); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); + Result read_bytes = input_stream_->Read(static_cast(out), nbytes, position); if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } + if (storage_read_bytes_) { + storage_read_bytes_->fetch_add(static_cast(read_bytes.value())); + } return read_bytes.value(); } @@ -106,12 +113,12 @@ arrow::Result> ArrowInputStreamAdapter::ReadAt(in arrow::Future> ArrowInputStreamAdapter::ReadAsync( const arrow::io::IOContext& io_context, int64_t position, int64_t nbytes) { auto fut = arrow::Future>::Make(); - auto range_status = ValidateArrowIoRange(position, "position"); + auto range_status = ValidateArrowIoRange(position, "position"); if (!range_status.ok()) { fut.MarkFinished(range_status); return fut; } - range_status = ValidateArrowIoRange(nbytes, "nbytes"); + range_status = ValidateArrowIoRange(nbytes, "nbytes"); if (!range_status.ok()) { fut.MarkFinished(range_status); return fut; @@ -124,15 +131,19 @@ arrow::Future> ArrowInputStreamAdapter::ReadAsync return fut; } std::shared_ptr buffer = std::move(buffer_result).ValueUnsafe(); - input_stream_->ReadAsync(reinterpret_cast(buffer->mutable_data()), - static_cast(nbytes), static_cast(position), - [fut, buffer](Status callback_status) mutable { - if (callback_status.ok()) { - fut.MarkFinished(std::move(buffer)); - } else { - fut.MarkFinished(ToArrowStatus(callback_status)); - } - }); + std::shared_ptr> storage_read_bytes = storage_read_bytes_; + input_stream_->ReadAsync( + reinterpret_cast(buffer->mutable_data()), nbytes, position, + [fut, buffer, storage_read_bytes, nbytes](Status callback_status) mutable { + if (callback_status.ok()) { + if (storage_read_bytes) { + storage_read_bytes->fetch_add(static_cast(nbytes)); + } + fut.MarkFinished(std::move(buffer)); + } else { + fut.MarkFinished(ToArrowStatus(callback_status)); + } + }); return fut; } @@ -145,7 +156,7 @@ arrow::Result ArrowInputStreamAdapter::Tell() const { } arrow::Result ArrowInputStreamAdapter::GetSize() { - return static_cast(file_size_); + return file_size_; } bool ArrowInputStreamAdapter::closed() const { diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h index 00ee1648..1c649108 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -33,7 +34,7 @@ class InputStream; class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile { public: ArrowInputStreamAdapter(const std::shared_ptr& input_stream, - const std::shared_ptr& pool, uint64_t file_size); + int64_t file_size, const std::shared_ptr& pool); ~ArrowInputStreamAdapter() override; // NOTE: In paimon file system definition, position + nbytes should not exceed file_size_. @@ -52,12 +53,21 @@ class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile } bool closed() const override; + // Accumulated bytes read from the underlying stream (storageReadBytes). The counter is owned + // by this adapter and initialized to 0; callers may retain the returned shared_ptr to read the + // value after the adapter is closed or destroyed. + const std::shared_ptr>& StorageReadBytes() const { + return storage_read_bytes_; + } + private: arrow::Status DoClose(); std::shared_ptr input_stream_; std::shared_ptr pool_; - uint64_t file_size_; + int64_t file_size_; + // Accumulates the number of bytes read from the underlying stream (storageReadBytes). + std::shared_ptr> storage_read_bytes_; bool closed_ = false; }; diff --git a/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp index c74c090f..f89e0810 100644 --- a/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp @@ -22,7 +22,6 @@ #include "arrow/result.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/result.h" @@ -51,15 +50,17 @@ bool ArrowOutputStreamAdapter::closed() const { } arrow::Status ArrowOutputStreamAdapter::Write(const void* data, int64_t nbytes) { - if (!InRange(nbytes)) { - return arrow::Status::Invalid( - fmt::format("nbytes value {} is out of bound of uint32_t", nbytes)); + if (nbytes < 0) { + return arrow::Status::Invalid(fmt::format("write size {} is less than 0", nbytes)); } - Result len = - out_->Write(static_cast(data), static_cast(nbytes)); + Result len = out_->Write(static_cast(data), nbytes); if (!len.ok()) { return ToArrowStatus(len.status()); } + if (len.value() != nbytes) { + return arrow::Status::IOError( + fmt::format("expect write len {} mismatch actual write len {}", nbytes, len.value())); + } return arrow::Status::OK(); } diff --git a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp index 04fba953..b5680607 100644 --- a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp @@ -60,9 +60,9 @@ TEST(ArrowStreamAdapterTest, TestInputAndOutputStream) { // in stream ASSERT_OK_AND_ASSIGN(std::shared_ptr in, file_system->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = - std::make_unique(in, GetArrowPool(GetDefaultPool()), length); + std::make_unique(in, length, GetArrowPool(GetDefaultPool())); ASSERT_EQ(in_stream->GetSize().ValueOrDie(), static_cast(data.length())); ASSERT_EQ(in_stream->Tell().ValueOrDie(), 0); ASSERT_FALSE(in_stream->closed()); diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index 88623b3f..de6aedbb 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -21,11 +21,32 @@ #include "arrow/array/array_base.h" #include "arrow/array/array_nested.h" +#include "arrow/array/concatenate.h" #include "arrow/util/compression.h" +#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" namespace paimon { + +namespace { + +bool HasNonZeroOffset(const std::shared_ptr& data) { + if (data->offset != 0) { + return true; + } + for (const auto& child : data->child_data) { + if (HasNonZeroOffset(child)) { + return true; + } + } + return false; +} + +} // namespace + +const char* ArrowUtils::kArrowSchemaMetadataKey = "ARROW:schema"; + Result> ArrowUtils::DataTypeToSchema( const std::shared_ptr& data_type) { if (data_type->id() != arrow::Type::STRUCT) { @@ -166,6 +187,28 @@ Result> ArrowUtils::RemoveFieldFromStructArr return array; } +Result> ArrowUtils::NormalizeRecordBatchOffsets( + const std::shared_ptr& record_batch, arrow::MemoryPool* pool) { + arrow::ArrayVector normalized_columns; + for (int32_t i = 0; i < record_batch->num_columns(); ++i) { + const std::shared_ptr& column = record_batch->column(i); + if (!HasNonZeroOffset(column->data())) { + continue; + } + if (normalized_columns.empty()) { + normalized_columns = record_batch->columns(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr normalized_column, + arrow::Concatenate({column}, pool)); + normalized_columns[i] = std::move(normalized_column); + } + if (normalized_columns.empty()) { + return record_batch; + } + return arrow::RecordBatch::Make(record_batch->schema(), record_batch->num_rows(), + std::move(normalized_columns)); +} + Result ArrowUtils::GetCompressionType(const std::string& compression) { std::string normalized = StringUtils::ToLowerCase(compression); if (normalized.empty() || normalized == "none") { diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h index f6dea3d1..1d4b7855 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.h +++ b/src/paimon/common/utils/arrow/arrow_utils.h @@ -23,7 +23,6 @@ #include "arrow/api.h" #include "arrow/util/type_fwd.h" -#include "fmt/format.h" #include "paimon/result.h" namespace paimon { @@ -33,6 +32,8 @@ class PAIMON_EXPORT ArrowUtils { ArrowUtils() = delete; ~ArrowUtils() = delete; + static const char* kArrowSchemaMetadataKey; + static Result> DataTypeToSchema( const std::shared_ptr& data_type); @@ -50,6 +51,9 @@ class PAIMON_EXPORT ArrowUtils { static Result> RemoveFieldFromStructArray( const std::shared_ptr& struct_array, const std::string& field_name); + static Result> NormalizeRecordBatchOffsets( + const std::shared_ptr& record_batch, arrow::MemoryPool* pool); + static bool EqualsIgnoreNullable(const std::shared_ptr& type, const std::shared_ptr& other_type); diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index 8908f679..a2f8d5bc 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" #include "paimon/testing/utils/testharness.h" + namespace paimon::test { TEST(ArrowUtilsTest, TestCreateProjection) { @@ -381,6 +382,54 @@ TEST(ArrowUtilsTest, TestRemoveFieldFromStructArraySuccess) { ASSERT_TRUE(result->Equals(expected_struct_array)); } +TEST(ArrowUtilsTest, TestNormalizeRecordBatchOffsets) { + auto value_field = arrow::field("value", arrow::int32()); + auto nested_field = arrow::field("nested", arrow::struct_({value_field})); + auto text_field = arrow::field("text", arrow::utf8()); + auto clean_field = arrow::field("clean", arrow::boolean()); + auto schema = arrow::schema({nested_field, text_field, clean_field}); + + std::shared_ptr values = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1, 2, 3, 4]").ValueOrDie(); + std::shared_ptr sliced_values = values->Slice(1, 3); + std::shared_ptr nested_column = + arrow::StructArray::Make({sliced_values}, {value_field->name()}).ValueOrDie(); + std::shared_ptr text = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c", "d", "e"])") + .ValueOrDie(); + std::shared_ptr sliced_text = text->Slice(1, 3); + std::shared_ptr clean_column = + arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), "[true, false, true]") + .ValueOrDie(); + std::shared_ptr record_batch = arrow::RecordBatch::Make( + schema, /*num_rows=*/3, {nested_column, sliced_text, clean_column}); + + ASSERT_EQ(nested_column->offset(), 0); + ASSERT_EQ(nested_column->field(0)->offset(), 1); + ASSERT_EQ(sliced_text->offset(), 1); + ASSERT_EQ(clean_column->offset(), 0); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr normalized_batch, + ArrowUtils::NormalizeRecordBatchOffsets(record_batch, arrow::default_memory_pool())); + ASSERT_NE(normalized_batch.get(), record_batch.get()); + ASSERT_TRUE(normalized_batch->Equals(*record_batch)); + std::shared_ptr normalized_nested = + std::static_pointer_cast(normalized_batch->column(0)); + ASSERT_EQ(normalized_nested->offset(), 0); + ASSERT_EQ(normalized_nested->field(0)->offset(), 0); + ASSERT_EQ(normalized_batch->column(1)->offset(), 0); + ASSERT_EQ(normalized_batch->column(2)->offset(), 0); + ASSERT_NE(normalized_batch->column_data(0).get(), record_batch->column_data(0).get()); + ASSERT_NE(normalized_batch->column_data(1).get(), record_batch->column_data(1).get()); + ASSERT_EQ(normalized_batch->column_data(2).get(), record_batch->column_data(2).get()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr unchanged_batch, + ArrowUtils::NormalizeRecordBatchOffsets(normalized_batch, arrow::default_memory_pool())); + ASSERT_EQ(unchanged_batch.get(), normalized_batch.get()); +} + TEST(ArrowUtilsTest, TestEqualsIgnoreNullable) { { // test simple @@ -483,6 +532,7 @@ TEST(ArrowUtilsTest, TestGetCompressionType) { ASSERT_EQ(type, arrow::Compression::GZIP); } { + // test invalid codec ASSERT_NOK(ArrowUtils::GetCompressionType("invalid_codec")); } } diff --git a/src/paimon/common/utils/arrow/mem_utils.cpp b/src/paimon/common/utils/arrow/mem_utils.cpp index 4ecb644f..7e8986be 100644 --- a/src/paimon/common/utils/arrow/mem_utils.cpp +++ b/src/paimon/common/utils/arrow/mem_utils.cpp @@ -21,10 +21,12 @@ #include #include +#include #include #include "arrow/memory_pool.h" #include "arrow/status.h" +#include "fmt/format.h" #include "paimon/memory/memory_pool.h" namespace paimon { @@ -35,14 +37,35 @@ class ArrowMemPoolAdaptor : public arrow::MemoryPool { : pool_(*pool), life_holder_(pool) {} arrow::Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override { - *out = reinterpret_cast(pool_.Malloc(size, alignment)); + uint8_t* new_out = nullptr; + try { + new_out = reinterpret_cast(pool_.Malloc(size, alignment)); + } catch (const std::bad_alloc&) { + return arrow::Status::OutOfMemory(fmt::format("failed to allocate {} bytes", size)); + } + if (size > 0 && new_out == nullptr) { + return arrow::Status::OutOfMemory(fmt::format("failed to allocate {} bytes", size)); + } + *out = new_out; stats_.DidAllocateBytes(size); return arrow::Status::OK(); } arrow::Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, uint8_t** ptr) override { - *ptr = reinterpret_cast(pool_.Realloc(*ptr, old_size, new_size, alignment)); + uint8_t* new_ptr = nullptr; + try { + new_ptr = + reinterpret_cast(pool_.Realloc(*ptr, old_size, new_size, alignment)); + } catch (const std::bad_alloc&) { + return arrow::Status::OutOfMemory( + fmt::format("failed to reallocate memory from {} to {} bytes", old_size, new_size)); + } + if (new_size > 0 && new_ptr == nullptr) { + return arrow::Status::OutOfMemory( + fmt::format("failed to reallocate memory from {} to {} bytes", old_size, new_size)); + } + *ptr = new_ptr; stats_.DidReallocateBytes(old_size, new_size); return arrow::Status::OK(); } diff --git a/src/paimon/common/utils/arrow/mem_utils.h b/src/paimon/common/utils/arrow/mem_utils.h index b59c4031..96b59e3e 100644 --- a/src/paimon/common/utils/arrow/mem_utils.h +++ b/src/paimon/common/utils/arrow/mem_utils.h @@ -26,9 +26,6 @@ #include "paimon/visibility.h" namespace paimon { -class MemoryPool; - -PAIMON_EXPORT std::unique_ptr GetArrowPool(MemoryPool& pool); PAIMON_EXPORT std::unique_ptr GetArrowPool( const std::shared_ptr& pool); diff --git a/src/paimon/common/utils/arrow/mem_utils_test.cpp b/src/paimon/common/utils/arrow/mem_utils_test.cpp index d5d28a7a..2e9bda22 100644 --- a/src/paimon/common/utils/arrow/mem_utils_test.cpp +++ b/src/paimon/common/utils/arrow/mem_utils_test.cpp @@ -19,10 +19,53 @@ #include "paimon/common/utils/arrow/mem_utils.h" +#include +#include +#include + +#include "arrow/status.h" #include "gtest/gtest.h" #include "paimon/memory/memory_pool.h" namespace paimon::test { +namespace { + +// A MemoryPool whose allocations always fail, either by returning nullptr or by +// throwing std::bad_alloc, so the adaptor's out-of-memory paths can be exercised. +class FailingMemoryPool : public MemoryPool { + public: + enum class Mode { kReturnNull, kThrowBadAlloc }; + + explicit FailingMemoryPool(Mode mode) : mode_(mode) {} + + void* Malloc(uint64_t /*size*/, uint64_t /*alignment*/ = 0) override { + if (mode_ == Mode::kThrowBadAlloc) { + throw std::bad_alloc(); + } + return nullptr; + } + + void* Realloc(void* /*p*/, size_t /*old_size*/, size_t /*new_size*/, + uint64_t /*alignment*/ = 0) override { + if (mode_ == Mode::kThrowBadAlloc) { + throw std::bad_alloc(); + } + return nullptr; + } + + void Free(void* /*p*/, uint64_t /*size*/) override {} + uint64_t CurrentUsage() const override { + return 0; + } + uint64_t MaxMemoryUsage() const override { + return 0; + } + + private: + Mode mode_; +}; + +} // namespace TEST(MemUtilsTest, TestSimple) { const int64_t alignment = 64; @@ -74,4 +117,32 @@ TEST(MemUtilsTest, TestSimple) { ASSERT_EQ(50, pool->max_memory()); } +TEST(MemUtilsTest, TestAllocateOutOfMemory) { + uint8_t* ptr = nullptr; + + // Underlying pool returns nullptr for a positive size. + auto null_pool = + GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kReturnNull)); + ASSERT_TRUE(null_pool->Allocate(16, 64, &ptr).IsOutOfMemory()); + + // Underlying pool throws std::bad_alloc. + auto throw_pool = + GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kThrowBadAlloc)); + ASSERT_TRUE(throw_pool->Allocate(16, 64, &ptr).IsOutOfMemory()); +} + +TEST(MemUtilsTest, TestReallocateOutOfMemory) { + uint8_t* ptr = nullptr; + + // Underlying pool returns nullptr for a positive new size. + auto null_pool = + GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kReturnNull)); + ASSERT_TRUE(null_pool->Reallocate(/*old_size=*/0, /*new_size=*/16, 64, &ptr).IsOutOfMemory()); + + // Underlying pool throws std::bad_alloc. + auto throw_pool = + GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kThrowBadAlloc)); + ASSERT_TRUE(throw_pool->Reallocate(/*old_size=*/0, /*new_size=*/16, 64, &ptr).IsOutOfMemory()); +} + } // namespace paimon::test diff --git a/src/paimon/common/utils/arrow/status_utils.h b/src/paimon/common/utils/arrow/status_utils.h index 40ac94cb..a0b505f6 100644 --- a/src/paimon/common/utils/arrow/status_utils.h +++ b/src/paimon/common/utils/arrow/status_utils.h @@ -99,10 +99,10 @@ inline Status ToPaimonStatus(const arrow::Status& status) { auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).ok(), ToPaimonStatus((result_name).status()), \ PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).ValueUnsafe(); + lhs = std::move(result_name).ValueUnsafe() #define PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL_FROM_ARROW( \ - PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)); + PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)) } // namespace paimon diff --git a/src/paimon/common/utils/binary_row_partition_computer.cpp b/src/paimon/common/utils/binary_row_partition_computer.cpp index b9b0685e..43ec7d40 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer.cpp @@ -139,13 +139,13 @@ Result BinaryRowPartitionComputer::GetTypeFromArrowSchema( Result BinaryRowPartitionComputer::PartToSimpleString( const std::shared_ptr& partition_type, const BinaryRow& partition, - const std::string& delimiter, int32_t max_length) { + const std::string& delimiter, int32_t max_length, bool legacy_partition_name_enabled) { std::vector partition_converters; partition_converters.reserve(partition_type->num_fields()); for (const auto& field : partition_type->fields()) { PAIMON_ASSIGN_OR_RAISE(DataConverterUtils::BinaryRowFieldToStrConverter converter, DataConverterUtils::CreateBinaryRowFieldToStringConverter( - field->type()->id(), /*legacy_partition_name_enabled=*/true)); + field->type()->id(), legacy_partition_name_enabled)); partition_converters.emplace_back(converter); } std::vector partition_vec; diff --git a/src/paimon/common/utils/binary_row_partition_computer.h b/src/paimon/common/utils/binary_row_partition_computer.h index ae371bf5..63aa7549 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.h +++ b/src/paimon/common/utils/binary_row_partition_computer.h @@ -63,7 +63,8 @@ class BinaryRowPartitionComputer { static Result PartToSimpleString( const std::shared_ptr& partition_type, const BinaryRow& partition, - const std::string& delimiter, int32_t max_length); + const std::string& delimiter, int32_t max_length, + bool legacy_partition_name_enabled = true); private: BinaryRowPartitionComputer(const std::vector& partition_keys, diff --git a/src/paimon/common/utils/decimal_utils.cpp b/src/paimon/common/utils/decimal_utils.cpp index ff4aa1bf..2cee2561 100644 --- a/src/paimon/common/utils/decimal_utils.cpp +++ b/src/paimon/common/utils/decimal_utils.cpp @@ -20,6 +20,7 @@ #include "paimon/common/utils/decimal_utils.h" #include +#include #include #include @@ -74,27 +75,51 @@ std::optional DecimalUtils::RescaleDecimalWithOverflowCheck( Result DecimalUtils::StrToInt128(const std::string& str) { try { - Decimal::int128_t ret = 0; size_t length = str.length(); - if (length > 0) { - bool is_negative = str[0] == '-'; - size_t posn = is_negative ? 1 : 0; - while (posn < length) { - size_t group = std::min(18ul, length - posn); - int64_t chunk = std::stoll(str.substr(posn, group)); - int64_t multiple = 1; - for (size_t i = 0; i < group; ++i) { - multiple *= 10; + if (length == 0) { + return Status::Invalid("invalid string: [], cannot convert to int128"); + } + bool is_negative = str[0] == '-'; + size_t posn = is_negative ? 1 : 0; + if (posn == length) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); + } + + Decimal::uint128_t magnitude = 0; + Decimal::uint128_t max_magnitude = (static_cast(1) << 127) - 1; + if (is_negative) { + max_magnitude += 1; + } + while (posn < length) { + size_t group = std::min(18ul, length - posn); + for (size_t i = 0; i < group; ++i) { + if (!std::isdigit(static_cast(str[posn + i]))) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); } - ret *= multiple; - ret += chunk; - posn += group; } - if (is_negative) { - ret = -ret; + uint64_t chunk = std::stoull(str.substr(posn, group)); + uint64_t multiple = 1; + for (size_t i = 0; i < group; ++i) { + multiple *= 10; + } + if (magnitude > (max_magnitude - chunk) / multiple) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); + } + magnitude = magnitude * multiple + chunk; + posn += group; + } + if (is_negative) { + if (magnitude == max_magnitude) { + auto max_value = + static_cast((static_cast(1) << 127) - 1); + return -max_value - 1; } + return -static_cast(magnitude); } - return ret; + return static_cast(magnitude); } catch (...) { return Status::Invalid(fmt::format("invalid string: [{}], cannot convert to int128", str)); } diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h index 77da7db8..72246769 100644 --- a/src/paimon/common/utils/field_type_utils.h +++ b/src/paimon/common/utils/field_type_utils.h @@ -26,6 +26,7 @@ #include "arrow/api.h" #include "arrow/type_fwd.h" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" @@ -49,6 +50,15 @@ class FieldTypeUtils { (type == FieldType::BIGINT); } + /// Converts an arrow field to a `FieldType`, disambiguating metadata-marked extension types + /// (a VARIANT field is physically a STRUCT with the variant metadata marker). + static Result ConvertToFieldType(const std::shared_ptr& field) { + if (VariantTypeUtils::IsVariantField(field)) { + return FieldType::VARIANT; + } + return ConvertToFieldType(field->type()->id()); + } + static Result ConvertToFieldType(const arrow::Type::type& arrow_type) { switch (arrow_type) { case arrow::Type::type::BOOL: @@ -123,6 +133,8 @@ class FieldTypeUtils { return "MAP"; case FieldType::STRUCT: return "STRUCT"; + case FieldType::VARIANT: + return "VARIANT"; default: return "UNKNOWN, type id:" + std::to_string(static_cast(type)); } diff --git a/src/paimon/common/utils/field_type_utils_test.cpp b/src/paimon/common/utils/field_type_utils_test.cpp index 7c2b237a..50f60237 100644 --- a/src/paimon/common/utils/field_type_utils_test.cpp +++ b/src/paimon/common/utils/field_type_utils_test.cpp @@ -102,8 +102,7 @@ TEST(FieldTypeUtilsTest, ConvertToFieldType) { ASSERT_EQ(result, FieldType::STRUCT); // Test unsupported Arrow type - ASSERT_NOK(FieldTypeUtils::ConvertToFieldType( - static_cast(999))); // Invalid Arrow type + ASSERT_NOK(FieldTypeUtils::ConvertToFieldType(arrow::Type::type::UINT8)); } // Test case: FieldTypeToString @@ -127,8 +126,8 @@ TEST(FieldTypeUtilsTest, FieldTypeToString) { ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::STRUCT), "STRUCT"); // Test UNKNOWN type - auto unknown_type = static_cast(999); - ASSERT_EQ(FieldTypeUtils::FieldTypeToString(unknown_type), "UNKNOWN, type id:999"); + auto unknown_type = static_cast(128); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(unknown_type), "UNKNOWN, type id:128"); } } // namespace paimon::test diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 54a5cfd9..bb6f9f2f 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -166,7 +166,7 @@ Result FieldsComparator::CompareField( return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(input_type.get()); assert(decimal_type); diff --git a/src/paimon/common/utils/file_type_test.cpp b/src/paimon/common/utils/file_type_test.cpp index b9698130..9f18a70c 100644 --- a/src/paimon/common/utils/file_type_test.cpp +++ b/src/paimon/common/utils/file_type_test.cpp @@ -31,6 +31,14 @@ TEST(FileTypeTest, TestIsIndex) { ASSERT_FALSE(FileTypeUtils::IsIndex(FileType::kData)); } +TEST(FileTypeTest, TestToString) { + ASSERT_EQ(FileTypeUtils::ToString(FileType::kMeta), "meta"); + ASSERT_EQ(FileTypeUtils::ToString(FileType::kData), "data"); + ASSERT_EQ(FileTypeUtils::ToString(FileType::kBucketIndex), "bucket_index"); + ASSERT_EQ(FileTypeUtils::ToString(FileType::kGlobalIndex), "global_index"); + ASSERT_EQ(FileTypeUtils::ToString(FileType::kFileIndex), "file_index"); +} + TEST(FileTypeTest, TestMetaPrefix) { ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/snapshot/snapshot-1"), FileType::kMeta); ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/schema/schema-2"), FileType::kMeta); @@ -176,6 +184,12 @@ TEST(FileTypeTest, TestInvalidTempWrapperFallsBackToOriginalName) { // Too short -> should not unwrap. ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/snapshot/.x.tmp"), FileType::kData); + + // Long enough and ends with .tmp, but the char before the trailing 41-char suffix is not a + // dot -> should not unwrap. + ASSERT_EQ(FileTypeUtils::Classify( + "dfs://cluster/db/snapshot/.snapshot-1234567890123456789012345678901234.tmp"), + FileType::kData); } } // namespace paimon::test diff --git a/src/paimon/common/utils/linked_hash_map.h b/src/paimon/common/utils/linked_hash_map.h index f831756e..02e0beef 100644 --- a/src/paimon/common/utils/linked_hash_map.h +++ b/src/paimon/common/utils/linked_hash_map.h @@ -73,12 +73,13 @@ class LinkedHashMap { } IteratorType erase(const K& key) { - if (!map_.count(key)) { + auto map_iter = map_.find(key); + if (map_iter == map_.end()) { return order_.end(); } - auto iter = order_.erase(map_[key]); - map_.erase(key); - return iter; + IteratorType order_iter = map_iter->second; + map_.erase(map_iter); + return order_.erase(order_iter); } IteratorType insert(const K& key, const V& value) { diff --git a/src/paimon/common/utils/math.h b/src/paimon/common/utils/math.h index 575ed41e..54ad6cf7 100644 --- a/src/paimon/common/utils/math.h +++ b/src/paimon/common/utils/math.h @@ -34,6 +34,10 @@ #include #include +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/status.h" + namespace paimon { template @@ -61,6 +65,23 @@ constexpr bool InRange(From value) { } } +template +Status ValidateValueInRange(From value, const char* value_name) { + if (PAIMON_UNLIKELY(!InRange(value))) { + return Status::Invalid(fmt::format("{} {} is out of bound of type {}", value_name, value, + OptionsUtils::GetTypeName())); + } + return Status::OK(); +} + +template +Status ValidateValueNonNegative(T value, const char* value_name) { + if (PAIMON_UNLIKELY(value < 0)) { + return Status::Invalid(fmt::format("{} {} is less than 0", value_name, value)); + } + return Status::OK(); +} + // Swaps between big and little endian. Can be used in combination with the // little-endian encoding/decoding functions in coding_lean.h and coding.h to // encode/decode big endian. diff --git a/src/paimon/common/utils/math_test.cpp b/src/paimon/common/utils/math_test.cpp index ae4a5037..36df1cba 100644 --- a/src/paimon/common/utils/math_test.cpp +++ b/src/paimon/common/utils/math_test.cpp @@ -22,6 +22,7 @@ #include #include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -74,6 +75,15 @@ TEST(MathTest, InRange) { ASSERT_FALSE(InRange(std::numeric_limits::max())); ASSERT_TRUE(InRange(std::numeric_limits::max())); ASSERT_FALSE(InRange(static_cast(std::numeric_limits::max()) + 1)); + + ASSERT_OK(ValidateValueInRange( + static_cast(std::numeric_limits::lowest()), "signed value")); + ASSERT_NOK_WITH_MSG(ValidateValueInRange(-1, "negative value"), + "negative value -1 is out of bound of type"); + + ASSERT_OK(ValidateValueNonNegative(0, "non-negative value")); + ASSERT_NOK_WITH_MSG(ValidateValueNonNegative(-1, "negative value"), + "negative value -1 is less than 0"); } } // namespace paimon::test diff --git a/src/paimon/common/utils/murmurhash_utils.h b/src/paimon/common/utils/murmurhash_utils.h index 52831c9a..18430f44 100644 --- a/src/paimon/common/utils/murmurhash_utils.h +++ b/src/paimon/common/utils/murmurhash_utils.h @@ -60,8 +60,8 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" namespace paimon { @@ -184,7 +184,9 @@ class MurmurHashUtils { int32_t length_aligned = length_in_bytes - length_in_bytes % 4; int32_t h1 = HashBytesByInt(segment, offset, length_aligned, seed); for (int32_t i = length_aligned; i < length_in_bytes; i++) { - int32_t k1 = MixK1(segment.Get(offset + i)); + auto byte = static_cast(segment.Get(offset + i)); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + int32_t k1 = MixK1(signed_byte); h1 = MixH1(h1, k1); } return Fmix(h1, length_in_bytes); @@ -240,10 +242,10 @@ class MurmurHashUtils { return value; } - static char GetByte(const void* base, int64_t offset) { - char value; - std::memcpy(&value, static_cast(base) + offset, sizeof(char)); - return value; + static int32_t GetByte(const void* base, int64_t offset) { + uint8_t value; + std::memcpy(&value, static_cast(base) + offset, sizeof(uint8_t)); + return value < 128 ? value : static_cast(value) - 256; } public: diff --git a/src/paimon/common/utils/read_ahead_cache.cpp b/src/paimon/common/utils/read_ahead_cache.cpp index aa7e5cb6..b0001189 100644 --- a/src/paimon/common/utils/read_ahead_cache.cpp +++ b/src/paimon/common/utils/read_ahead_cache.cpp @@ -28,6 +28,7 @@ #include #include "paimon/common/utils/byte_range_combiner.h" +#include "paimon/common/utils/math.h" namespace paimon { @@ -126,18 +127,13 @@ Status ReadAheadCache::Impl::Init(std::vector&& ranges) { if (is_initialized_) { return Status::Invalid("Cache has already been initialized"); } - if (config_.GetRangeSizeLimit() > static_cast(std::numeric_limits::max())) { - return Status::Invalid("CacheConfig range_size_limit exceeds uint32_t max"); - } - PAIMON_ASSIGN_OR_RAISE( std::vector pending_ranges, ByteRangeCombiner::CoalesceByteRanges(std::move(ranges), config_.GetHoleSizeLimit(), config_.GetRangeSizeLimit())); for (const auto& pending_range : pending_ranges) { - if (pending_range.length > static_cast(std::numeric_limits::max())) { - return Status::Invalid("range length should not be larger than uint32_t max"); - } + PAIMON_RETURN_NOT_OK(ValidateValueInRange(pending_range.offset, "range offset")); + PAIMON_RETURN_NOT_OK(ValidateValueInRange(pending_range.length, "range length")); } pending_ranges_ = pending_ranges; is_cached_ = std::vector>(pending_ranges_.size()); @@ -228,8 +224,10 @@ std::vector ReadAheadCache::Impl::MakeCacheEntries( auto promise = std::make_shared>(); auto future = promise->get_future(); auto buffer = std::make_shared(range.length, memory_pool_.get()); + auto read_size = static_cast(buffer->size()); + auto read_offset = static_cast(range.offset); stream_->ReadAsync( - buffer->data(), static_cast(buffer->size()), range.offset, + buffer->data(), read_size, read_offset, [promise, buffer](Status status) mutable { promise->set_value(status); }); new_entries.emplace_back(range, std::move(buffer), std::move(future)); } diff --git a/src/paimon/common/utils/roaring_bitmap32.cpp b/src/paimon/common/utils/roaring_bitmap32.cpp index af5214b4..8a90952a 100644 --- a/src/paimon/common/utils/roaring_bitmap32.cpp +++ b/src/paimon/common/utils/roaring_bitmap32.cpp @@ -312,4 +312,33 @@ RoaringBitmap32::Iterator RoaringBitmap32::EqualOrLarger(int32_t key) const { return iter; } +std::optional RoaringBitmap32::NextValue(int32_t x) const { + auto iter = EqualOrLarger(x); + if (iter == End()) { + return std::nullopt; + } + return *iter; +} + +std::optional RoaringBitmap32::PreviousValue(int32_t x) const { + if (IsEmpty()) { + return std::nullopt; + } + + auto& bitmap = GetRoaringBitmap(roaring_bitmap_); + if (x <= static_cast(bitmap.minimum())) { + return std::nullopt; + } + + const uint64_t rank = bitmap.rank(static_cast(x - 1)); + if (rank == 0) { + return std::nullopt; + } + + uint32_t value = 0; + [[maybe_unused]] bool found = bitmap.select(static_cast(rank - 1), &value); + assert(found); + return static_cast(value); +} + } // namespace paimon diff --git a/src/paimon/common/utils/roaring_bitmap32_test.cpp b/src/paimon/common/utils/roaring_bitmap32_test.cpp index 008ae259..239381f2 100644 --- a/src/paimon/common/utils/roaring_bitmap32_test.cpp +++ b/src/paimon/common/utils/roaring_bitmap32_test.cpp @@ -256,6 +256,25 @@ TEST(RoaringBitmap32Test, TestIterator) { ASSERT_EQ(iter, roaring.End()); } +TEST(RoaringBitmap32Test, TestNextAndPreviousValue) { + RoaringBitmap32 roaring = RoaringBitmap32::From({10, 20, 30}); + + ASSERT_EQ(roaring.NextValue(5), std::optional(10)); + ASSERT_EQ(roaring.NextValue(10), std::optional(10)); + ASSERT_EQ(roaring.NextValue(25), std::optional(30)); + ASSERT_EQ(roaring.NextValue(31), std::nullopt); + + ASSERT_EQ(roaring.PreviousValue(10), std::nullopt); + ASSERT_EQ(roaring.PreviousValue(11), std::optional(10)); + ASSERT_EQ(roaring.PreviousValue(20), std::optional(10)); + ASSERT_EQ(roaring.PreviousValue(21), std::optional(20)); + ASSERT_EQ(roaring.PreviousValue(100), std::optional(30)); + + RoaringBitmap32 empty; + ASSERT_EQ(empty.NextValue(0), std::nullopt); + ASSERT_EQ(empty.PreviousValue(0), std::nullopt); +} + TEST(RoaringBitmap32Test, TestIteratorAssignAndMove) { RoaringBitmap32 roaring = RoaringBitmap32::From({10, 100, 200}); diff --git a/src/paimon/common/utils/row_range_index.cpp b/src/paimon/common/utils/row_range_index.cpp index 87bff084..c65bcf20 100644 --- a/src/paimon/common/utils/row_range_index.cpp +++ b/src/paimon/common/utils/row_range_index.cpp @@ -33,11 +33,8 @@ RowRangeIndex::RowRangeIndex(std::vector ranges) : ranges_(std::move(rang } } -Result RowRangeIndex::Create(const std::vector& ranges) { - if (ranges.empty()) { - return Status::Invalid("Ranges cannot be empty in RowRangeIndex"); - } - return RowRangeIndex(Range::SortAndMergeOverlap(ranges, /*adjacent=*/true)); +Result RowRangeIndex::Create(const std::vector& ranges, bool merge_adjacent) { + return RowRangeIndex(Range::SortAndMergeOverlap(ranges, merge_adjacent)); } const std::vector& RowRangeIndex::Ranges() const { @@ -49,6 +46,17 @@ bool RowRangeIndex::Intersects(int64_t start, int64_t end) const { return candidate < static_cast(starts_.size()) && starts_[candidate] <= end; } +bool RowRangeIndex::Contains(const Range& range) const { + int32_t candidate = LowerBound(range.from); + return candidate < static_cast(ranges_.size()) && + ranges_[candidate].from <= range.from && ranges_[candidate].to >= range.to; +} + +bool RowRangeIndex::ContainsExactly(const Range& range) const { + int32_t candidate = LowerBound(range.from); + return candidate < static_cast(ranges_.size()) && ranges_[candidate] == range; +} + std::vector RowRangeIndex::IntersectedRanges(int64_t start, int64_t end) const { int32_t left = LowerBound(start); if (left >= static_cast(ranges_.size())) { diff --git a/src/paimon/common/utils/row_range_index_test.cpp b/src/paimon/common/utils/row_range_index_test.cpp index e9095f64..d2d755a8 100644 --- a/src/paimon/common/utils/row_range_index_test.cpp +++ b/src/paimon/common/utils/row_range_index_test.cpp @@ -25,8 +25,9 @@ namespace paimon::test { // ======================== Create ======================== -TEST(RowRangeIndexTest, CreateWithEmptyRangesReturnsError) { - ASSERT_NOK_WITH_MSG(RowRangeIndex::Create({}), "Ranges cannot be empty in RowRangeIndex"); +TEST(RowRangeIndexTest, CreateWithEmptyRangesReturnsEmptyIndex) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({})); + ASSERT_TRUE(index.Ranges().empty()); } TEST(RowRangeIndexTest, CreateWithSingleRange) { @@ -165,6 +166,66 @@ TEST(RowRangeIndexTest, IntersectsSinglePointNoMatch) { ASSERT_FALSE(index.Intersects(11, 11)); } +// ======================== Contains ======================== + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyExactRange) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20), Range(30, 40)})); + Range query(10, 20); + + ASSERT_TRUE(index.Contains(query)); + ASSERT_TRUE(index.ContainsExactly(query)); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyBoundarySubRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20)})); + + ASSERT_TRUE(index.Contains(Range(10, 19))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 19))); + + ASSERT_TRUE(index.Contains(Range(11, 20))); + ASSERT_FALSE(index.ContainsExactly(Range(11, 20))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyOutOfBoundaryRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20)})); + + ASSERT_FALSE(index.Contains(Range(9, 20))); + ASSERT_FALSE(index.ContainsExactly(Range(9, 20))); + + ASSERT_FALSE(index.Contains(Range(10, 21))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 21))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyRangeAcrossTwoDisjointRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + + ASSERT_FALSE(index.Contains(Range(10, 11))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 11))); + ASSERT_FALSE(index.ContainsExactly(Range(0, 20))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyAdjacentMergeFlagDifference) { + ASSERT_OK_AND_ASSIGN(auto merged_index, + RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, true)); + ASSERT_OK_AND_ASSIGN(auto non_merged_index, + RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + + Range query(10, 11); + ASSERT_TRUE(merged_index.Contains(query)); + ASSERT_FALSE(merged_index.ContainsExactly(query)); + + ASSERT_FALSE(non_merged_index.Contains(query)); + ASSERT_FALSE(non_merged_index.ContainsExactly(query)); +} + +// ======================== merge_adjacent ======================== + +TEST(RowRangeIndexTest, CreateDoesNotMergeAdjacentRangesWhenDisabled) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + std::vector expected = {Range(0, 10), Range(11, 20)}; + ASSERT_EQ(index.Ranges(), expected); +} + // ======================== IntersectedRanges ======================== TEST(RowRangeIndexTest, IntersectedRangesExactMatch) { diff --git a/src/paimon/common/utils/serialization_utils.h b/src/paimon/common/utils/serialization_utils.h index 4718ea92..c1e97d03 100644 --- a/src/paimon/common/utils/serialization_utils.h +++ b/src/paimon/common/utils/serialization_utils.h @@ -27,7 +27,6 @@ #include "fmt/format.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" @@ -35,6 +34,7 @@ #include "paimon/macros.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" diff --git a/src/paimon/common/utils/status_test.cpp b/src/paimon/common/utils/status_test.cpp index 68fb5c41..cd9b60fe 100644 --- a/src/paimon/common/utils/status_test.cpp +++ b/src/paimon/common/utils/status_test.cpp @@ -74,6 +74,43 @@ TEST(StatusTest, TestToStringWithDetail) { ASSERT_EQ(status.ToString(), ss.str()); } +TEST(StatusTest, TestCodeAsString) { + ASSERT_EQ("OK", Status::CodeAsString(StatusCode::OK)); + ASSERT_EQ("Out of memory", Status::CodeAsString(StatusCode::OutOfMemory)); + ASSERT_EQ("Key error", Status::CodeAsString(StatusCode::KeyError)); + ASSERT_EQ("Type error", Status::CodeAsString(StatusCode::TypeError)); + ASSERT_EQ("Invalid", Status::CodeAsString(StatusCode::Invalid)); + ASSERT_EQ("IOError", Status::CodeAsString(StatusCode::IOError)); + ASSERT_EQ("Capacity error", Status::CodeAsString(StatusCode::CapacityError)); + ASSERT_EQ("Index error", Status::CodeAsString(StatusCode::IndexError)); + ASSERT_EQ("Cancelled", Status::CodeAsString(StatusCode::Cancelled)); + ASSERT_EQ("Unknown error", Status::CodeAsString(StatusCode::UnknownError)); + ASSERT_EQ("NotImplemented", Status::CodeAsString(StatusCode::NotImplemented)); + ASSERT_EQ("Serialization error", Status::CodeAsString(StatusCode::SerializationError)); + ASSERT_EQ("Not exist", Status::CodeAsString(StatusCode::NotExist)); + ASSERT_EQ("Exist", Status::CodeAsString(StatusCode::Exist)); + + // An out-of-range code falls into the default branch. The cast is intentional to + // exercise the defensive default, so the enum-range analyzer check is suppressed. + // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) + ASSERT_EQ("Unknown", Status::CodeAsString(static_cast(99))); + + // The instance overload returns "OK" for a success status. + ASSERT_EQ("OK", Status::OK().CodeAsString()); +} + +TEST(StatusDeathTest, TestAbort) { + // Death tests fork(); with the default "fast" style, forking in a multi-threaded + // process is unsafe and under ThreadSanitizer the child aborts with a sanitizer + // message before Abort() runs. The "threadsafe" style re-execs the test binary in + // a clean process so Abort()'s own output is produced and can be matched. + const std::string prev_style = testing::GTEST_FLAG(death_test_style); + testing::GTEST_FLAG(death_test_style) = "threadsafe"; + ASSERT_DEATH(Status::IOError("boom").Abort(), "Paimon Fatal Error"); + ASSERT_DEATH(Status::IOError("boom").Abort("custom prefix"), "custom prefix"); + testing::GTEST_FLAG(death_test_style) = prev_style; +} + TEST(StatusTest, TestWithDetail) { Status status(StatusCode::IOError, "summary"); auto detail = std::make_shared(); diff --git a/src/paimon/common/utils/stream_utils.h b/src/paimon/common/utils/stream_utils.h index 5537c83b..b8b67088 100644 --- a/src/paimon/common/utils/stream_utils.h +++ b/src/paimon/common/utils/stream_utils.h @@ -46,11 +46,11 @@ class StreamUtils { static Result> ReadFully(std::unique_ptr input_stream, const std::shared_ptr& pool) { PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); PAIMON_UNIQUE_PTR content = Bytes::AllocateBytes(file_length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, - input_stream->Read(content->data(), content->size())); - if (static_cast(actual_read_len) != file_length) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, + input_stream->Read(content->data(), file_length)); + if (actual_read_len != file_length) { return Status::Invalid("actual read length {}, not match with expect length {}", actual_read_len, file_length); } @@ -59,7 +59,7 @@ class StreamUtils { static Result> ReadAsyncFully( std::unique_ptr input_stream, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); PAIMON_UNIQUE_PTR content = Bytes::AllocateBytes(file_length, pool.get()); PAIMON_RETURN_NOT_OK(ReadAsyncFully(std::move(input_stream), content->data())); return content; @@ -67,10 +67,10 @@ class StreamUtils { static Status ReadAsyncFully(std::unique_ptr input_stream, char* content) { PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); - uint64_t read_offset = 0; - uint32_t read_len = std::min(file_length, kDefaultReadChunkSize); + int64_t read_offset = 0; + int64_t read_len = std::min(file_length, kDefaultReadChunkSize); std::vector> futures; futures.reserve(file_length / kDefaultReadChunkSize + 1); while (read_len > 0) { @@ -96,7 +96,7 @@ class StreamUtils { } private: - static constexpr uint64_t kDefaultReadChunkSize = 1024 * 1024; + static constexpr int64_t kDefaultReadChunkSize = 1024 * 1024; }; } // namespace paimon diff --git a/src/paimon/common/utils/stream_utils_test.cpp b/src/paimon/common/utils/stream_utils_test.cpp index 9786a3eb..74c92fe9 100644 --- a/src/paimon/common/utils/stream_utils_test.cpp +++ b/src/paimon/common/utils/stream_utils_test.cpp @@ -49,8 +49,8 @@ class StreamUtilsTest : public ::testing::Test { std::string CreateTestFile(const std::string& content) { std::string file_path = dir_->Str() + "/test_file.txt"; EXPECT_OK_AND_ASSIGN(auto output_stream, file_system_->Create(file_path, true)); - EXPECT_OK_AND_ASSIGN(int32_t length, output_stream->Write(content.data(), content.size())); - EXPECT_EQ(length, static_cast(content.size())); + EXPECT_OK_AND_ASSIGN(int64_t length, output_stream->Write(content.data(), content.size())); + EXPECT_EQ(length, static_cast(content.size())); EXPECT_OK(output_stream->Close()); return file_path; } diff --git a/src/paimon/common/utils/vector_store_utils.h b/src/paimon/common/utils/vector_store_utils.h new file mode 100644 index 00000000..c50c0967 --- /dev/null +++ b/src/paimon/common/utils/vector_store_utils.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Utils for vector-store files. +/// +/// A vector-store file is identified by the `.vector.` marker in its name. +class VectorStoreUtils { + public: + VectorStoreUtils() = delete; + ~VectorStoreUtils() = delete; + + /// Returns true if `file_name` is a vector-store file (contains the `.vector.` marker). + static bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; + } +}; + +} // namespace paimon diff --git a/src/paimon/core/append/append_compact_coordinator.cpp b/src/paimon/core/append/append_compact_coordinator.cpp index 318a9c66..37693a58 100644 --- a/src/paimon/core/append/append_compact_coordinator.cpp +++ b/src/paimon/core/append/append_compact_coordinator.cpp @@ -137,7 +137,8 @@ Result> CreateFileStoreScan( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, pool)); + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, ManifestFile::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 3edc12a4..feea29c5 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -19,6 +19,8 @@ #include "paimon/core/append/append_only_writer.h" #include +#include +#include #include #include @@ -26,33 +28,32 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/types/row_kind.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/long_counter.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/append_data_file_writer_factory.h" +#include "paimon/core/io/blob_data_file_writer_factory.h" #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" -#include "paimon/core/io/data_file_writer.h" #include "paimon/core/io/data_increment.h" #include "paimon/core/io/multiple_blob_file_writer.h" #include "paimon/core/io/rolling_blob_file_writer.h" #include "paimon/core/io/rolling_file_writer.h" +#include "paimon/core/io/shredding_append_data_file_writer_factory.h" #include "paimon/core/io/single_file_writer.h" #include "paimon/core/manifest/file_source.h" +#include "paimon/core/operation/blob_file_context.h" #include "paimon/core/utils/commit_increment.h" -#include "paimon/format/file_format.h" -#include "paimon/format/file_format_factory.h" -#include "paimon/format/writer_builder.h" #include "paimon/macros.h" #include "paimon/metrics.h" #include "paimon/record_batch.h" namespace paimon { -class MemoryPool; -class FormatStatsExtractor; - AppendOnlyWriter::AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& write_schema, const std::optional>& write_cols, @@ -84,6 +85,21 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { if (writer_ == nullptr) { PAIMON_ASSIGN_OR_RAISE(writer_, CreateRollingRowWriter()); } + + if (!inline_descriptor_fields_.empty() || !inline_view_fields_.empty()) { + auto data_type = arrow::struct_(write_schema_->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(batch->GetData(), data_type)); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields( + struct_array, inline_descriptor_fields_, "blob-descriptor-field")); + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields(struct_array, inline_view_fields_, + "blob-view-field")); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); + return writer_->Write(&c_array); + } + return writer_->Write(batch->GetData()); } @@ -155,70 +171,70 @@ Status AppendOnlyWriter::Flush(bool wait_for_latest_compaction, bool forced_full return Status::OK(); } -AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWriter() const { - auto schemas = BlobUtils::SeparateBlobSchema(write_schema_); - if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) { - return CreateRollingBlobWriter(schemas); - } else { - return std::make_unique>>( - options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(write_schema_, write_cols_)); +AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWriter() { + auto blob_context = BlobFileContext::Create(write_schema_, options_); + + // Save inline descriptor and view fields for validation in Write() + if (blob_context) { + inline_descriptor_fields_ = blob_context->GetDescriptorFields(); + inline_view_fields_ = blob_context->GetViewFields(); + } + + if (blob_context && blob_context->RequireBlobFileWriter()) { + // Use context-aware schema separation: inline BLOB fields stay in main + auto schemas = + BlobUtils::SeparateBlobSchema(write_schema_, blob_context->GetInlineFields()); + return CreateRollingBlobWriter(schemas, blob_context->GetInlineFields()); } + + // No BLOB fields, or all BLOB fields are inline and no .blob files are needed. + PAIMON_ASSIGN_OR_RAISE(WriterFactory writer_factory, + GetDataFileWriterFactory(write_schema_, write_cols_)); + return std::make_unique>>( + options_.GetTargetFileSize(/*has_primary_key=*/false), options_.GetTargetFileRowNum(), + writer_factory); } -AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetDataFileWriterCreator( +Result AppendOnlyWriter::GetDataFileWriterFactory( const std::shared_ptr& schema, const std::optional>& write_cols) const { - return - [this, schema, write_cols]() - -> Result< - std::unique_ptr>>> { - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); - auto format = options_.GetFileFormat(); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(memory_pool_); - - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - auto writer = std::make_unique( - options_.GetFileCompression(), std::function(), - schema_id_, seq_num_counter_, FileSource::Append(), stats_extractor, - path_factory_->IsExternalPath(), write_cols, memory_pool_); - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); - return writer; - }; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan_factory, + ShreddingWritePlanFactories::SelectActive(options_, schema, memory_pool_)); + if (plan_factory != nullptr) { + return std::make_shared( + options_, schema_id_, schema, write_cols, seq_num_counter_, FileSource::Append(), + path_factory_, plan_factory, memory_pool_); + } + return std::make_shared(options_, schema_id_, schema, write_cols, + seq_num_counter_, FileSource::Append(), + path_factory_, memory_pool_); } -AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetBlobFileWriterCreator( - const std::shared_ptr& writer_builder, - const std::shared_ptr& stats_extractor, - const std::optional>& write_cols) const { - return - [this, writer_builder, stats_extractor, write_cols]() - -> Result< - std::unique_ptr>>> { - auto writer = std::make_unique( - /*compression=*/"none", std::function(), - schema_id_, seq_num_counter_, FileSource::Append(), stats_extractor, - path_factory_->IsExternalPath(), write_cols, memory_pool_); - PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), - path_factory_->NewBlobPath(), writer_builder)); - return writer; - }; +AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( + const std::shared_ptr& single_field_schema, + const std::optional>& write_cols, bool write_placeholder) const { + std::shared_ptr path_factory = path_factory_; + return std::make_shared(options_, schema_id_, single_field_schema, + write_cols, seq_num_counter_, path_factory, + write_placeholder, memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( - const BlobUtils::SeparatedSchemas& schemas) const { + const BlobUtils::SeparatedSchemas& schemas, const std::set& inline_fields) const { // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; - auto blob_writer_creator = [this, blob_schema](const std::string& blob_field_name) + // A data-evolution write touching only blob columns is a partial update: its rows may mark + // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any + // write that also carries non-blob columns stores blob bytes verbatim. The gate cannot tell + // a first write from an update, so a blob-only first write also runs under the sentinel + // channel: a user value equal to the sentinel is then persisted as a placeholder entry with + // no older layer to resolve it, and reading fails loudly rather than returning wrong bytes. + bool write_placeholder = + options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0; + MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = + [this, blob_schema, write_placeholder](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -228,30 +244,25 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri fmt::format("Blob field '{}' not found in blob schema", blob_field_name)); } auto single_field_schema = arrow::schema({field}); - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(memory_pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - std::vector write_cols = {blob_field_name}; - auto single_blob_file_writer_creator = - GetBlobFileWriterCreator(writer_builder, stats_extractor, write_cols); + auto single_blob_file_writer_factory = + GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder); return std::make_unique>>( - options_.GetBlobTargetFileSize(), single_blob_file_writer_creator); + options_.GetBlobTargetFileSize(), + /*target_file_row_num=*/std::numeric_limits::max(), + single_blob_file_writer_factory); }; + WriterFactory main_writer_factory; + if (schemas.main_schema->num_fields() > 0) { + PAIMON_ASSIGN_OR_RAISE( + main_writer_factory, + GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names())); + } return std::make_unique( - options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(schemas.main_schema, schemas.main_schema->field_names()), - blob_schema, blob_writer_creator, arrow::struct_(write_schema_->fields())); + options_.GetTargetFileSize(/*has_primary_key=*/false), options_.GetTargetFileRowNum(), + main_writer_factory, blob_schema, blob_writer_creator, + arrow::struct_(write_schema_->fields()), inline_fields); } Status AppendOnlyWriter::Sync() { @@ -282,5 +293,4 @@ Status AppendOnlyWriter::Close() { } return Status::OK(); } - } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 25febf7c..2bfb1a53 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -19,9 +19,10 @@ #pragma once #include -#include +#include #include #include +#include #include #include @@ -29,7 +30,6 @@ #include "paimon/core/compact/compact_manager.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" -#include "paimon/core/io/single_file_writer.h" #include "paimon/core/utils/batch_writer.h" #include "paimon/result.h" #include "paimon/status.h" @@ -47,12 +47,12 @@ class CommitIncrement; class RecordBatch; template class RollingFileWriter; +template +class SingleFileWriterFactory; class LongCounter; class DataFilePathFactory; class MemoryPool; class Metrics; -class FormatStatsExtractor; -class WriterBuilder; class AppendOnlyWriter : public BatchWriter { public: @@ -91,26 +91,26 @@ class AppendOnlyWriter : public BatchWriter { } private: - using SingleFileWriterCreator = std::function< - Result>>>()>; + using WriterFactory = + std::shared_ptr>>; using RollingFileWriterResult = Result>>>; - RollingFileWriterResult CreateRollingRowWriter() const; + RollingFileWriterResult CreateRollingRowWriter(); RollingFileWriterResult CreateRollingBlobWriter( - const BlobUtils::SeparatedSchemas& schemas) const; + const BlobUtils::SeparatedSchemas& schemas, + const std::set& inline_fields) const; Result DrainIncrement(); Status Flush(bool wait_for_latest_compaction, bool forced_full_compaction); - SingleFileWriterCreator GetDataFileWriterCreator( + Result GetDataFileWriterFactory( const std::shared_ptr& schema, const std::optional>& write_cols) const; - SingleFileWriterCreator GetBlobFileWriterCreator( - const std::shared_ptr& writer_builder, - const std::shared_ptr& stats_extractor, - const std::optional>& write_cols) const; + WriterFactory GetBlobFileWriterFactory( + const std::shared_ptr& single_field_schema, + const std::optional>& write_cols, bool write_placeholder) const; Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); @@ -132,6 +132,8 @@ class AppendOnlyWriter : public BatchWriter { std::shared_ptr compact_deletion_file_; std::unique_ptr>> writer_; + std::set inline_descriptor_fields_; + std::set inline_view_fields_; }; } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index db1adb87..068398b3 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -33,10 +33,17 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/fs/external_path_provider.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/core/compact/compact_deletion_file.h" #include "paimon/core/compact/compact_result.h" #include "paimon/core/compact/noop_compact_manager.h" @@ -48,10 +55,13 @@ #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/defs.h" +#include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/record_batch.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" namespace arrow { @@ -237,7 +247,81 @@ class AppendOnlyWriterTest : public testing::Test { return batch_builder.Finish().value(); } - private: + /// Creates a RecordBatch from a JSON string matching the given schema. + std::unique_ptr CreateBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto struct_type = arrow::struct_(schema->fields()); + auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.Finish().value(); + } + + /// Opens a file using the specified format and returns a reader. + std::unique_ptr OpenFormatReader(const std::string& file_path, + const std::string& format) const { + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_TRUE(input_stream); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format, /*options=*/{})); + EXPECT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + return reader_builder->Build(input_stream).value(); + } + + /// Reads a file's content and compares it to expected_array. + void CheckFileContent(const std::string& file_path, const std::string& format, + const std::shared_ptr& expected_array) const { + auto reader = OpenFormatReader(file_path, format); + auto c_file_schema = reader->GetFileSchema().value(); + ASSERT_OK(reader->SetReadSchema(c_file_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); + ASSERT_TRUE(expected_array->Equals(result_array)) + << "Expected:\n" + << expected_array->ToString() << "\nActual:\n" + << result_array->ToString(); + } + + /// Reads a file's schema, compares structure against expected physical schema + /// (ignoring metadata), then verifies shared-shredding map metadata on the given field. + void CheckShreddingFileSchema(const std::string& file_path, const std::string& format, + const std::shared_ptr& expected_physical_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta, + const std::string& compression) const { + auto reader = OpenFormatReader(file_path, format); + auto c_file_schema = reader->GetFileSchema().value(); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Compare schema structure (types + field names), ignoring metadata. + ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_physical_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + + // Deserialize and compare the per-field shared-shredding map metadata. + auto metadata = file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN(auto deserialized_meta, + MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy())); + ASSERT_EQ(expected_meta, deserialized_meta); + } + + Result> CreateAppendOnlyWriter( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& memory_pool) const { + return std::make_unique(options, schema_id, write_schema, write_cols, + max_sequence_number, path_factory, + compact_manager, memory_pool); + } + + protected: std::shared_ptr memory_pool_; std::shared_ptr compact_manager_; }; @@ -261,11 +345,12 @@ TEST_F(AppendOnlyWriterTest, TestEmptyCommits) { ASSERT_TRUE(dir); ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); for (int32_t i = 0; i < 3; i++) { - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(true)); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(true)); ASSERT_TRUE(inc.GetNewFilesIncrement().IsEmpty()); ASSERT_TRUE(inc.GetCompactIncrement().IsEmpty()); } @@ -290,9 +375,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/2, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/2, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { ASSERT_TRUE(builder.Append(std::to_string(j)).ok()); @@ -302,9 +388,9 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { ASSERT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.Finish()); - ASSERT_OK(writer.Write(std::move(record_batch))); + ASSERT_OK(writer->Write(std::move(record_batch))); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(true)); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(true)); ASSERT_FALSE(inc.GetNewFilesIncrement().IsEmpty()); const auto& data_increment = inc.GetNewFilesIncrement(); const auto& data_file_metas = data_increment.NewFiles(); @@ -314,7 +400,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { std::string path = path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); ASSERT_OK_AND_ASSIGN(bool exist, options.GetFileSystem()->Exists(path)); ASSERT_TRUE(exist); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { @@ -332,9 +418,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), {std::make_shared()}); @@ -351,9 +438,9 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.Finish()); - ASSERT_OK(writer.Write(std::move(record_batch))); + ASSERT_OK(writer->Write(std::move(record_batch))); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); auto file_system = std::make_shared(); std::vector> file_status_list; @@ -376,9 +463,10 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), {std::make_shared()}); @@ -394,10 +482,10 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.SetRowKinds({RecordBatch::RowKind::DELETE}).Finish()); - ASSERT_NOK_WITH_MSG(writer.Write(std::move(record_batch)), + ASSERT_NOK_WITH_MSG(writer->Write(std::move(record_batch)), "Append only writer can not accept record batch with RowKind DELETE"); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); auto file_system = std::make_shared(); std::vector> file_status_list; @@ -414,17 +502,18 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitWaitCompactionUsesBlockingGetResul arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a", "b"}))); - ASSERT_OK(writer.PrepareCommit(/*wait_compaction=*/true).status()); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a", "b"}))); + ASSERT_OK(writer->PrepareCommit(/*wait_compaction=*/true).status()); ASSERT_EQ(compact_manager->get_result_blocking_calls.size(), 2); ASSERT_FALSE(compact_manager->get_result_blocking_calls[0]); ASSERT_TRUE(compact_manager->get_result_blocking_calls[1]); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestPrepareCommitForceCompactUsesBlockingGetResult) { @@ -436,17 +525,18 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitForceCompactUsesBlockingGetResult) arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a"}))); - ASSERT_OK(writer.PrepareCommit(/*wait_compaction=*/false).status()); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a"}))); + ASSERT_OK(writer->PrepareCommit(/*wait_compaction=*/false).status()); ASSERT_EQ(compact_manager->get_result_blocking_calls.size(), 2); ASSERT_FALSE(compact_manager->get_result_blocking_calls[0]); ASSERT_TRUE(compact_manager->get_result_blocking_calls[1]); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, @@ -479,13 +569,14 @@ TEST_F(AppendOnlyWriterTest, arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Sync()); - ASSERT_OK(writer.Sync()); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Sync()); + ASSERT_OK(writer->Sync()); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/false)); ASSERT_EQ(inc.GetCompactIncrement().CompactBefore().size(), 2); ASSERT_EQ(inc.GetCompactIncrement().CompactAfter().size(), 2); @@ -498,7 +589,7 @@ TEST_F(AppendOnlyWriterTest, ASSERT_TRUE(merged); ASSERT_EQ(merged->Id(), "d2"); ASSERT_EQ(merged->MergedOld(), deletion_file1); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { @@ -522,13 +613,14 @@ TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Sync()); + ASSERT_OK(writer->Sync()); ASSERT_TRUE(options.GetFileSystem()->Exists(compact_after_path).value()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); ASSERT_FALSE(options.GetFileSystem()->Exists(compact_after_path).value()); ASSERT_TRUE(compact_manager->request_cancel_called); ASSERT_TRUE(compact_manager->wait_called); @@ -554,15 +646,16 @@ TEST_F(AppendOnlyWriterTest, TestCloseCleansDeletionFile) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); // Sync to consume the compaction result and populate compact_deletion_file_. - ASSERT_OK(writer.Sync()); + ASSERT_OK(writer->Sync()); ASSERT_FALSE(deletion_file->Cleaned()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); ASSERT_TRUE(deletion_file->Cleaned()); } @@ -576,14 +669,15 @@ TEST_F(AppendOnlyWriterTest, TestCompactNotCompletedTriggersCompaction) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK_AND_ASSIGN(bool not_completed, writer.CompactNotCompleted()); + ASSERT_OK_AND_ASSIGN(bool not_completed, writer->CompactNotCompleted()); ASSERT_TRUE(not_completed); ASSERT_EQ(compact_manager->trigger_calls, std::vector({false})); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { @@ -595,19 +689,21 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Compact(/*full_compaction=*/true)); - ASSERT_OK(writer.Compact(/*full_compaction=*/false)); + ASSERT_OK(writer->Compact(/*full_compaction=*/true)); + ASSERT_OK(writer->Compact(/*full_compaction=*/false)); ASSERT_EQ(compact_manager->trigger_calls, std::vector({true, false})); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { - auto options = - CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::TARGET_FILE_ROW_NUM, "1"}}); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); auto path_factory = CreatePathFactory(dir->Str(), "orc", options); @@ -616,9 +712,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { auto blob_field = BlobUtils::ToArrowField("blob", false); auto schema = arrow::schema({int_field, blob_field}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::Int32Builder int_builder; ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); @@ -628,17 +725,66 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { ASSERT_TRUE(blob_builder.Append("bb", 2).ok()); auto blob_array = blob_builder.Finish().ValueOrDie(); - ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, blob_array}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Write( + CreateStructBatch(schema, {int_array->Slice(0, 1), blob_array->Slice(0, 1)}))); + ASSERT_OK(writer->Write( + CreateStructBatch(schema, {int_array->Slice(1, 1), blob_array->Slice(1, 1)}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + + const auto& files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(files.size(), 4); + for (const auto& file : files) { + ASSERT_EQ(file->row_count, 1); + ASSERT_TRUE(options.GetFileSystem()->Exists(path_factory->ToPath(file->file_name)).value()); + } + ASSERT_OK(writer->Close()); +} - ASSERT_EQ(inc.GetNewFilesIncrement().NewFiles().size(), 2); - const auto& main_file = inc.GetNewFilesIncrement().NewFiles()[0]; - const auto& blob_file = inc.GetNewFilesIncrement().NewFiles()[1]; - ASSERT_TRUE( - options.GetFileSystem()->Exists(path_factory->ToPath(main_file->file_name)).value()); - ASSERT_TRUE( - options.GetFileSystem()->Exists(path_factory->ToPath(blob_file->file_name)).value()); - ASSERT_OK(writer.Close()); +TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) { + auto options = + CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto blob_field = BlobUtils::ToArrowField("blob", false); + auto schema = arrow::schema({blob_field}); + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, schema, + /*write_cols=*/std::vector{"blob"}, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + arrow::LargeBinaryBuilder blob_builder; + ASSERT_TRUE(blob_builder.Append("a", 1).ok()); + ASSERT_TRUE(blob_builder.Append("bb", 2).ok()); + auto blob_array = blob_builder.Finish().ValueOrDie(); + + ASSERT_OK(writer->Write(CreateStructBatch(schema, {blob_array}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + const auto& new_files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(new_files.size(), 1); + ASSERT_TRUE(BlobUtils::IsBlobFile(new_files[0]->file_name)); + ASSERT_EQ(new_files[0]->row_count, 2); + ASSERT_TRUE(new_files[0]->write_cols.has_value()); + ASSERT_EQ(new_files[0]->write_cols.value(), std::vector({"blob"})); + std::string blob_file_path = path_factory->ToPath(new_files[0]->file_name); + ASSERT_TRUE(options.GetFileSystem()->Exists(blob_file_path).value()); + + auto blob_reader = OpenFormatReader(blob_file_path, "blob"); + ::ArrowSchema c_blob_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_blob_schema).ok()); + ASSERT_OK(blob_reader->SetReadSchema(&c_blob_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual_array, ReadResultCollector::CollectResult(blob_reader.get())); + auto expected_struct_array = + arrow::StructArray::Make({blob_array}, {blob_field->name()}).ValueOrDie(); + auto expected_array = std::make_shared(expected_struct_array); + ASSERT_TRUE(expected_array->Equals(actual_array)) << "Expected:\n" + << expected_array->ToString() << "\nActual:\n" + << actual_array->ToString(); } TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { @@ -651,9 +797,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { auto schema = arrow::schema({arrow::field("id", arrow::int32()), BlobUtils::ToArrowField("blob1", false), BlobUtils::ToArrowField("blob2", false)}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::Int32Builder int_builder; ASSERT_TRUE(int_builder.AppendValues({1}).ok()); @@ -665,8 +812,8 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { ASSERT_TRUE(blob_builder2.Append("b", 1).ok()); auto blob_array2 = blob_builder2.Finish().ValueOrDie(); - ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, blob_array1, blob_array2}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, blob_array1, blob_array2}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); ASSERT_EQ(inc.GetNewFilesIncrement().NewFiles().size(), 3); const auto& main_file = inc.GetNewFilesIncrement().NewFiles()[0]; @@ -678,7 +825,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { options.GetFileSystem()->Exists(path_factory->ToPath(blob_file1->file_name)).value()); ASSERT_TRUE( options.GetFileSystem()->Exists(path_factory->ToPath(blob_file2->file_name)).value()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { @@ -689,14 +836,15 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a", "b", "c"}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement first, writer.PrepareCommit(/*wait_compaction=*/false)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"d", "e"}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement second, writer.PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a", "b", "c"}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement first, writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"d", "e"}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement second, writer->PrepareCommit(/*wait_compaction=*/false)); ASSERT_EQ(first.GetNewFilesIncrement().NewFiles().size(), 1); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles().size(), 1); @@ -704,7 +852,967 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { ASSERT_EQ(first.GetNewFilesIncrement().NewFiles()[0]->max_sequence_number, 2); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles()[0]->min_sequence_number, 3); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles()[0]->max_sequence_number, 4); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); +} + +TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BLOB_VIEW_FIELD, "view"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto schema = + arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); + + // Build f0 column + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + // Build view column with valid BlobViewStruct values + arrow::LargeBinaryBuilder view_builder; + BlobViewStruct view_struct_0(Identifier("db", "tbl"), /*field_id=*/1, /*row_id=*/0); + auto view_bytes_0 = view_struct_0.Serialize(memory_pool_); + ASSERT_TRUE(view_builder.Append(view_bytes_0->data(), view_bytes_0->size()).ok()); + + BlobViewStruct view_struct_1(Identifier("db", "tbl"), /*field_id=*/1, /*row_id=*/1); + auto view_bytes_1 = view_struct_1.Serialize(memory_pool_); + ASSERT_TRUE(view_builder.Append(view_bytes_1->data(), view_bytes_1->size()).ok()); + + auto view_array = view_builder.Finish().ValueOrDie(); + ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, view_array}))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_FALSE(inc.GetNewFilesIncrement().NewFiles().empty()); + ASSERT_OK(writer->Close()); +} + +TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BLOB_VIEW_FIELD, "view"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto schema = + arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); + ASSERT_OK_AND_ASSIGN( + auto writer, CreateAppendOnlyWriter( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); + + // Build f0 column + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(1).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + // Build view column with raw bytes + arrow::LargeBinaryBuilder view_builder; + ASSERT_TRUE(view_builder.Append("not_a_valid_blob_view_or_descriptor").ok()); + auto view_array = view_builder.Finish().ValueOrDie(); + + ASSERT_NOK_WITH_MSG(writer->Write(CreateStructBatch(schema, {int_array, view_array})), + "BLOB inline field view require values to be set as corresponding type."); + ASSERT_OK(writer->Close()); +} + +/// Parameterized test class for shared-shredding tests, parameterized by file format. +class AppendOnlyWriterShreddingTest : public AppendOnlyWriterTest, + public ::testing::WithParamInterface { + public: + std::string GetFormat() const { + return GetParam(); + } +}; + +INSTANTIATE_TEST_SUITE_P(FileFormats, AppendOnlyWriterShreddingTest, + ::testing::Values("parquet", "orc")); + +TEST_F(AppendOnlyWriterTest, TestSharedShreddingMapRejectsAvroFormatOnCommit) { + auto options = CreateOptions({ + {Options::FILE_FORMAT, "avro"}, + {Options::MANIFEST_FORMAT, "avro"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "avro", options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_NOK_WITH_MSG(writer->PrepareCommit(/*wait_compaction=*/true), + "AddMetadata is not supported by avro format writer."); + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { + std::string format = GetFormat(); + // Configure with shared-shredding map on "tags" field, K=3. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {Options::TARGET_FILE_ROW_NUM, "2"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + // Logical schema: id(INT32), tags(MAP) + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Write a batch with MAP data using the logical schema. + // Row0: id=1, tags={a:10, b:20} → fits K=3 + // Row1: id=2, tags={c:30, a:40, b:50} → fits K=3 + // Row2: id=3, tags={a:60} → fits K=3 + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]], + [3, [["a", 60]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + // Verify we got one data file. + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // Check shared-shredding map metadata: a=0, b=1, c=2; K=3, max_row_width=3, no overflow. + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1, 2}}, {2, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + std::string compression = options.GetFileCompression(); + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, compression); + + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, 0, 1], 30, 40, 50, null]], + [3, [[0, -1, -1], 60, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, []], + [2, []] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map first_file_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); + MapSharedShreddingFieldMeta empty_meta; + empty_meta.num_columns = 3; + empty_meta.max_row_width = 0; + CheckShreddingFileSchema(data_file_path, format, first_schema, /*field_index=*/1, empty_meta, + options.GetFileCompression()); + + auto physical_type = arrow::struct_(first_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[-1, -1, -1], null, null, null, null]], + [2, [[-1, -1, -1], null, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyFiles) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {Options::TARGET_FILE_ROW_NUM, "2"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto null_batch = CreateBatch(logical_schema, R"([ + [1, null], + [2, null] + ])"); + ASSERT_OK(writer->Write(std::move(null_batch))); + + auto empty_batch = CreateBatch(logical_schema, R"([ + [3, []], + [4, []] + ])"); + ASSERT_OK(writer->Write(std::move(empty_batch))); + + auto null_value_batch = CreateBatch(logical_schema, R"([ + [5, [["a", null]]], + [6, [["b", null]]], + [7, [["c", 7], ["d", null]]] + ])"); + ASSERT_OK(writer->Write(std::move(null_value_batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + const auto& files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(3, files.size()); + std::string null_file_path = path_factory->ToPath(files[0]->file_name); + std::string empty_file_path = path_factory->ToPath(files[1]->file_name); + std::string null_value_file_path = path_factory->ToPath(files[2]->file_name); + + std::map first_file_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); + auto first_physical_type = arrow::struct_(first_schema->fields()); + + MapSharedShreddingFieldMeta empty_meta; + empty_meta.num_columns = 3; + empty_meta.max_row_width = 0; + CheckShreddingFileSchema(null_file_path, format, first_schema, /*field_index=*/1, empty_meta, + options.GetFileCompression()); + + std::shared_ptr expected_null_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(first_physical_type, {R"([ + [1, null], + [2, null] + ])"}, + &expected_null_array) + .ok()); + CheckFileContent(null_file_path, format, expected_null_array); + + std::map subsequent_file_k = {{"tags", 1}}; + ASSERT_OK_AND_ASSIGN(auto subsequent_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, subsequent_file_k)); + auto subsequent_physical_type = arrow::struct_(subsequent_schema->fields()); + empty_meta.num_columns = 1; + CheckShreddingFileSchema(empty_file_path, format, subsequent_schema, /*field_index=*/1, + empty_meta, options.GetFileCompression()); + + std::shared_ptr expected_empty_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(subsequent_physical_type, {R"([ + [3, [[-1], null, null]], + [4, [[-1], null, null]] + ])"}, + &expected_empty_array) + .ok()); + CheckFileContent(empty_file_path, format, expected_empty_array); + + MapSharedShreddingFieldMeta null_value_meta; + null_value_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}}; + null_value_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}}; + null_value_meta.overflow_field_set = {3}; + null_value_meta.num_columns = 1; + null_value_meta.max_row_width = 2; + CheckShreddingFileSchema(null_value_file_path, format, subsequent_schema, /*field_index=*/1, + null_value_meta, options.GetFileCompression()); + + std::shared_ptr expected_null_value_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(subsequent_physical_type, {R"([ + [5, [[0], null, null]], + [6, [[1], null, null]], + [7, [[2], 7, [[3, null]]]] + ])"}, + &expected_null_value_array) + .ok()); + CheckFileContent(null_value_file_path, format, expected_null_value_array); + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { + std::string format = GetFormat(); + // K=2, write rows with 3+ keys to trigger overflow. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Row0: {a:1, b:2} → fits K=2 + // Row1: {c:3, a:4, b:5} → 3 keys, K=2: c→col0, a→col1, b→overflow + // Row2: {d:6, e:7, f:8, a:9} → 4 keys, K=2: d→col0, e→col1, f+a→overflow + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 1], ["b", 2]]], + [2, [["c", 3], ["a", 4], ["b", 5]]], + [3, [["d", 6], ["e", 7], ["f", 8], ["a", 9]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + std::string compression = options.GetFileCompression(); + + // Verify metadata: a=0,b=1,c=2,d=3,e=4,f=5; K=2, max_row_width=4 + // b overflows in row1, f and a overflow in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}, {"f", 5}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}, {3, {0}}, {4, {1}}}; + expected_meta.overflow_field_set = {0, 1, 5}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 4; + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, compression); + + // Verify data content. + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1], 1, 2, null]], + [2, [[2, 0], 3, 4, [[1, 5]]]], + [3, [[3, 4], 6, 7, [[5, 8], [0, 9]]]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacement) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "lru"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 40], ["b", 50]]], + [3, [["d", 60]]], + [4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {2}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 4; + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, options.GetFileCompression()); + + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, 2], 10, 20, 30, null]], + [2, [[0, 1, -1], 40, 50, null, null]], + [3, [[-1, -1, 3], null, null, 60, null]], + [4, [[0, 1, 3], 70, 80, 100, [[2, 90]]]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossRollingFiles) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {Options::TARGET_FILE_ROW_NUM, "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // --- File 1: max_row_width = 3, K = K_max = 10 (first file, no history) --- + auto batch1 = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch1))); + + // File 2 adapts to file 1's max row width. + auto batch2 = CreateBatch(logical_schema, R"([ + [3, [["x", 100], ["y", 200], ["z", 300], ["w", 400], ["v", 500]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch2))); + + // File 3 sees both preceding files in the same rolling session. + auto batch3 = CreateBatch(logical_schema, R"([ + [4, [["p", 1000], ["q", 2000], ["r", 3000], ["s", 4000]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch3))); + ASSERT_OK_AND_ASSIGN(CommitIncrement increment, + writer->PrepareCommit(/*wait_compaction=*/true)); + const auto& files = increment.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(3, files.size()); + std::string file1_path = path_factory->ToPath(files[0]->file_name); + std::string file2_path = path_factory->ToPath(files[1]->file_name); + std::string file3_path = path_factory->ToPath(files[2]->file_name); + + std::map column_to_k_file1 = {{"tags", 10}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file1)); + MapSharedShreddingFieldMeta meta1; + meta1.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta1.field_to_columns = {{0, {0, 1}}, {1, {1, 2}}, {2, {0}}}; + meta1.num_columns = 10; + meta1.max_row_width = 3; + std::string compression = options.GetFileCompression(); + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1, + compression); + + std::map column_to_k_file2 = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file2)); + MapSharedShreddingFieldMeta meta2; + meta2.name_to_id = {{"x", 0}, {"y", 1}, {"z", 2}, {"w", 3}, {"v", 4}}; + meta2.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}}; + meta2.overflow_field_set = {3, 4}; + meta2.num_columns = 3; + meta2.max_row_width = 5; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2, + compression); + auto physical_type2 = arrow::struct_(phys_schema2->fields()); + std::shared_ptr expected_array2; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type2, {R"([ + [3, [[0, 1, 2], 100, 200, 300, [[3, 400], [4, 500]]]] + ])"}, + &expected_array2) + .ok()); + CheckFileContent(file2_path, format, expected_array2); + + std::map column_to_k_file3 = {{"tags", 5}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file3)); + MapSharedShreddingFieldMeta meta3; + meta3.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}, {"s", 3}}; + meta3.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {3}}}; + meta3.num_columns = 5; + meta3.max_row_width = 4; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/1, meta3, + compression); + auto physical_type3 = arrow::struct_(phys_schema3->fields()); + std::shared_ptr expected_array3; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type3, {R"([ + [4, [[0, 1, 2, 3, -1], 1000, 2000, 3000, 4000, null, null]] + ])"}, + &expected_array3) + .ok()); + CheckFileContent(file3_path, format, expected_array3); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapFirstFileUsesConfiguredMaxK) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + auto writer = std::make_unique( + options, /*schema_id=*/0, logical_schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + + std::string file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 10}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k)); + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}}; + expected_meta.num_columns = 10; + expected_meta.max_row_width = 3; + CheckShreddingFileSchema(file_path, format, physical_schema, /*field_index=*/1, expected_meta, + options.GetFileCompression()); + + auto physical_type = arrow::struct_(physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, 2, -1, -1, -1, -1, -1, -1, -1], + 10, 20, 30, null, null, null, null, null, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(file_path, format, expected_array); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsAdaptAcrossRollingFiles) { + std::string format = GetFormat(); + // Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4). + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {Options::TARGET_FILE_ROW_NUM, "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "8"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {"fields.attrs.map.storage-layout", "shared-shredding"}, + {"fields.attrs.map.shared-shredding.max-columns", "4"}, + {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + std::string compression = options.GetFileCompression(); + + // --- File 1: first file, tags K=8, attrs K=4 --- + // tags: max_row_width=2, attrs: max_row_width=1 + auto batch1 = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]], [["x", "v1"]]], + [2, [["a", 30]], [["x", "v2"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch1))); + + auto batch2 = CreateBatch(logical_schema, R"([ + [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch2))); + + auto batch3 = CreateBatch(logical_schema, R"([ + [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch3))); + ASSERT_OK_AND_ASSIGN(CommitIncrement increment, + writer->PrepareCommit(/*wait_compaction=*/true)); + const auto& files = increment.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(3, files.size()); + std::string file1_path = path_factory->ToPath(files[0]->file_name); + std::string file2_path = path_factory->ToPath(files[1]->file_name); + std::string file3_path = path_factory->ToPath(files[2]->file_name); + + std::map col_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file1)); + MapSharedShreddingFieldMeta meta1_tags; + meta1_tags.name_to_id = {{"a", 0}, {"b", 1}}; + meta1_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta1_tags.num_columns = 8; + meta1_tags.max_row_width = 2; + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1_tags, + compression); + MapSharedShreddingFieldMeta meta1_attrs; + meta1_attrs.name_to_id = {{"x", 0}}; + meta1_attrs.field_to_columns = {{0, {0}}}; + meta1_attrs.num_columns = 4; + meta1_attrs.max_row_width = 1; + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/2, meta1_attrs, + compression); + + std::map col_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file2)); + MapSharedShreddingFieldMeta meta2_tags; + meta2_tags.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; + meta2_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta2_tags.overflow_field_set = {2}; + meta2_tags.num_columns = 2; + meta2_tags.max_row_width = 3; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2_tags, + compression); + MapSharedShreddingFieldMeta meta2_attrs; + meta2_attrs.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}}; + meta2_attrs.field_to_columns = {{0, {0}}}; + meta2_attrs.overflow_field_set = {1, 2}; + meta2_attrs.num_columns = 1; + meta2_attrs.max_row_width = 3; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/2, meta2_attrs, + compression); + + std::map col_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file3)); + + MapSharedShreddingFieldMeta meta3_tags; + meta3_tags.name_to_id = {{"f", 0}, {"g", 1}}; + meta3_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta3_tags.num_columns = 3; + meta3_tags.max_row_width = 2; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/1, meta3_tags, + compression); + + MapSharedShreddingFieldMeta meta3_attrs; + meta3_attrs.name_to_id = {{"s", 0}, {"t", 1}}; + meta3_attrs.field_to_columns = {{0, {0}}, {1, {1}}}; + meta3_attrs.num_columns = 3; + meta3_attrs.max_row_width = 2; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/2, meta3_attrs, + compression); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { + std::string format = GetFormat(); + // Verify PrepareCommit returns correct DataFileMeta for shared-shredding map files. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/5, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/9, path_factory, + compact_manager_, memory_pool_)); + + // Write 3 rows. + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]], + [3, [["a", 40], ["b", 50], ["c", 60]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + + auto actual_meta = inc.GetNewFilesIncrement().NewFiles()[0]; + + // Construct expected value_stats independently. + // Physical schema has 2 top-level fields: id(INT32), tags(STRUCT). + // id: min=1, max=3, null_count=0; tags is nested: NullType(), null_count=null. + int32_t map_null_count = (format == "parquet" ? -1 : 0); + auto expected_value_stats = BinaryRowGenerator::GenerateStats( + {1, NullType()}, {3, NullType()}, {0, map_null_count}, memory_pool_.get()); + + // Build expected DataFileMeta with fake file_name/file_size (TEST_Equal ignores them). + auto expected_meta = DataFileMeta::ForAppend( + /*file_name=*/"fake-file.parquet", /*file_size=*/999, /*row_count=*/3, + expected_value_stats, + /*min_sequence_number=*/10, /*max_sequence_number=*/12, + /*schema_id=*/5, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt) + .value(); + + ASSERT_TRUE(expected_meta->TEST_Equal(*actual_meta)); + + // Verify the written file has correct shared-shredding map content. + std::string file_path = path_factory->ToPath(actual_meta->file_name); + std::map col_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k)); + auto physical_type = arrow::struct_(phys_schema->fields()); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, -1, -1], 30, null, null, null]], + [3, [[0, 1, 2], 40, 50, 60, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(file_path, format, expected_array); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) { + std::string format = GetFormat(); + // Schema: id(INT32), blob_data(BLOB), tags(MAP) + // BLOB field will be separated into a .blob file; MAP field uses shared-shredding. + // This tests that blob separation + shredding works correctly together. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + BlobUtils::ToArrowField("blob_data", false), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Write rows with id, blob_data, and tags. + // Row0: id=1, blob="hello", tags={a:10, b:20} + // Row1: id=2, blob="world", tags={c:30} + auto batch = CreateBatch(logical_schema, R"([ + [1, "hello", [["a", 10], ["b", 20]]], + [2, "world", [["c", 30]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + // Verify: should produce 2 files — one main data file and one .blob file. + const auto& new_files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(2, new_files.size()); + + // Identify main vs blob file. + std::string main_file_path, blob_file_path; + for (const auto& file_meta : new_files) { + std::string file_path = path_factory->ToPath(file_meta->file_name); + if (BlobUtils::IsBlobFile(file_meta->file_name)) { + blob_file_path = file_path; + } else { + main_file_path = file_path; + } + } + ASSERT_FALSE(main_file_path.empty()) << "Main data file not found"; + ASSERT_FALSE(blob_file_path.empty()) << "Blob file not found"; + + // Verify both files exist on disk. + auto fs = options.GetFileSystem(); + ASSERT_TRUE(fs->Exists(main_file_path).value()); + ASSERT_TRUE(fs->Exists(blob_file_path).value()); + + // Verify main file schema: should have id(INT32) + tags(shredded STRUCT), no blob_data. + // Build expected physical schema for the main schema (id + tags). + auto main_logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map col_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k)); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 2; + + CheckShreddingFileSchema(main_file_path, format, expected_physical_schema, + /*field_index=*/1, expected_meta, options.GetFileCompression()); + + // Verify main file content: id + shredded tags. + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, -1, -1], 30, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(main_file_path, format, expected_array); + + // Verify blob file exists and can be read (blob_data column). + auto blob_reader = OpenFormatReader(blob_file_path, "blob"); + ASSERT_NE(blob_reader, nullptr); } } // namespace paimon::test diff --git a/src/paimon/core/bucket/bucket_id_calculator.cpp b/src/paimon/core/bucket/bucket_id_calculator.cpp index 6bede5ab..c3dcb79e 100644 --- a/src/paimon/core/bucket/bucket_id_calculator.cpp +++ b/src/paimon/core/bucket/bucket_id_calculator.cpp @@ -208,7 +208,7 @@ static Result WriteBucketRow(int32_t col_id, }; return writer_func; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { const auto* decimal_type = arrow::internal::checked_cast(field->type().get()); assert(decimal_type); diff --git a/src/paimon/core/bucket/bucket_select_converter_test.cpp b/src/paimon/core/bucket/bucket_select_converter_test.cpp index 94c2f60d..2707eccb 100644 --- a/src/paimon/core/bucket/bucket_select_converter_test.cpp +++ b/src/paimon/core/bucket/bucket_select_converter_test.cpp @@ -18,12 +18,14 @@ #include "paimon/core/bucket/bucket_select_converter.h" +#include #include #include #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/core/bucket/default_bucket_function.h" +#include "paimon/core/bucket/hive_bucket_function.h" #include "paimon/core/bucket/mod_bucket_function.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" @@ -36,41 +38,66 @@ namespace paimon::test { class BucketSelectConverterTest : public ::testing::Test { - protected: + public: + void AssertDefaultBucket(FieldType field_type, const Literal& literal, + const std::shared_ptr& arrow_type, + const BinaryRowGenerator::ValueType& values, + int32_t num_buckets = 17) const { + auto predicate = PredicateBuilder::Equal(0, "key", field_type, literal); + + ASSERT_OK_AND_ASSIGN( + std::optional selected_bucket, + BucketSelectConverter::Convert(predicate, {"key"}, {arrow_type}, + BucketFunctionType::DEFAULT, num_buckets, pool_.get())); + ASSERT_TRUE(selected_bucket.has_value()); + + BinaryRow row = BinaryRowGenerator::GenerateRow(values, pool_.get()); + DefaultBucketFunction function; + ASSERT_EQ(function.Bucket(row, num_buckets), selected_bucket.value()); + } + + private: std::shared_ptr pool_ = GetDefaultPool(); }; -TEST_F(BucketSelectConverterTest, SingleIntEqualDefault) { - int32_t num_buckets = 10; - Literal lit(static_cast(42)); - auto predicate = PredicateBuilder::Equal(0, "id", FieldType::INT, lit); - - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"id"}, {arrow::int32()}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); +TEST_F(BucketSelectConverterTest, SingleStringEqualDefault) { + std::string value = "hello_world"; + AssertDefaultBucket(FieldType::STRING, Literal(FieldType::STRING, value.c_str(), value.size()), + arrow::utf8(), {value}, 8); +} - // Verify by computing the expected bucket manually - auto row = BinaryRowGenerator::GenerateRow({static_cast(42)}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); +TEST_F(BucketSelectConverterTest, PrimitiveKeyTypes) { + AssertDefaultBucket(FieldType::BOOLEAN, Literal(true), arrow::boolean(), {true}); + AssertDefaultBucket(FieldType::TINYINT, Literal(static_cast(-12)), arrow::int8(), + {static_cast(-12)}); + AssertDefaultBucket(FieldType::SMALLINT, Literal(static_cast(1234)), arrow::int16(), + {static_cast(1234)}); + AssertDefaultBucket(FieldType::INT, Literal(static_cast(42)), arrow::int32(), + {static_cast(42)}, 10); + AssertDefaultBucket(FieldType::BIGINT, Literal(static_cast(123456789L)), + arrow::int64(), {static_cast(123456789L)}, 16); + AssertDefaultBucket(FieldType::FLOAT, Literal(1.25F), arrow::float32(), {1.25F}); + AssertDefaultBucket(FieldType::DOUBLE, Literal(-123.5), arrow::float64(), {-123.5}); } -TEST_F(BucketSelectConverterTest, SingleStringEqualDefault) { - int32_t num_buckets = 8; - std::string val = "hello_world"; - Literal lit(FieldType::STRING, val.c_str(), val.size()); - auto predicate = PredicateBuilder::Equal(0, "name", FieldType::STRING, lit); +TEST_F(BucketSelectConverterTest, TimestampMillisPrecision) { + // TIMESTAMP with millisecond precision (compact storage, precision=3) + Timestamp ts = Timestamp::FromEpochMillis(1700000000000L); + AssertDefaultBucket(FieldType::TIMESTAMP, Literal(ts), arrow::timestamp(arrow::TimeUnit::MILLI), + {TimestampType(ts, 3)}, 10); +} - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"name"}, {arrow::utf8()}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); +TEST_F(BucketSelectConverterTest, TimestampMicrosPrecision) { + // TIMESTAMP with microsecond precision (non-compact storage, precision=6) + Timestamp ts(1700000000000L, 123456); + AssertDefaultBucket(FieldType::TIMESTAMP, Literal(ts), arrow::timestamp(arrow::TimeUnit::MICRO), + {TimestampType(ts, 6)}, 10); +} - // Verify - auto row = BinaryRowGenerator::GenerateRow({val}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); +TEST_F(BucketSelectConverterTest, DecimalKey) { + Decimal decimal = Decimal::FromUnscaledLong(12345L, 10, 2); + AssertDefaultBucket(FieldType::DECIMAL, Literal(decimal), arrow::decimal128(10, 2), {decimal}, + 10); } TEST_F(BucketSelectConverterTest, MultiKeyAndPredicate) { @@ -175,22 +202,6 @@ TEST_F(BucketSelectConverterTest, NullPredicateReturnsNullopt) { ASSERT_FALSE(result.has_value()); } -TEST_F(BucketSelectConverterTest, BigintKeyDefault) { - int32_t num_buckets = 16; - Literal lit(static_cast(123456789L)); - auto predicate = PredicateBuilder::Equal(0, "user_id", FieldType::BIGINT, lit); - - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"user_id"}, {arrow::int64()}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); - - // Verify - auto row = BinaryRowGenerator::GenerateRow({static_cast(123456789L)}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); -} - TEST_F(BucketSelectConverterTest, AndWithExtraPredicateStillWorks) { // AND(EQUAL(id, 42), GREATER_THAN(value, 100)) // Only id is bucket key, value is not — should still derive bucket from id @@ -211,60 +222,52 @@ TEST_F(BucketSelectConverterTest, AndWithExtraPredicateStillWorks) { ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); } -TEST_F(BucketSelectConverterTest, TimestampMillisPrecision) { - // TIMESTAMP with millisecond precision (compact storage, precision=3) - int32_t num_buckets = 10; - Timestamp ts = Timestamp::FromEpochMillis(1700000000000L); - Literal lit(ts); - auto predicate = PredicateBuilder::Equal(0, "ts", FieldType::TIMESTAMP, lit); +TEST_F(BucketSelectConverterTest, HiveBucketFunctionWithDecimal) { + int32_t num_buckets = 11; + Decimal decimal = Decimal::FromUnscaledLong(12345L, 10, 2); + auto int_predicate = + PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(7))); + auto decimal_predicate = + PredicateBuilder::Equal(1, "amount", FieldType::DECIMAL, Literal(decimal)); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({int_predicate, decimal_predicate})); - auto arrow_type = arrow::timestamp(arrow::TimeUnit::MILLI); - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"ts"}, {arrow_type}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); - - // Verify: precision=3 uses compact WriteTimestamp - auto row = BinaryRowGenerator::GenerateRow({TimestampType(ts, 3)}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); + ASSERT_OK_AND_ASSIGN( + std::optional selected_bucket, + BucketSelectConverter::Convert(predicate, {"id", "amount"}, + {arrow::int32(), arrow::decimal128(10, 2)}, + BucketFunctionType::HIVE, num_buckets, pool_.get())); + ASSERT_TRUE(selected_bucket.has_value()); + + BinaryRow row = + BinaryRowGenerator::GenerateRow({static_cast(7), decimal}, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr function, + HiveBucketFunction::Create({HiveFieldInfo(FieldType::INT), + HiveFieldInfo(FieldType::DECIMAL, 10, 2)})); + ASSERT_EQ(function->Bucket(row, num_buckets), selected_bucket.value()); } -TEST_F(BucketSelectConverterTest, TimestampMicrosPrecision) { - // TIMESTAMP with microsecond precision (non-compact storage, precision=6) - int32_t num_buckets = 10; - Timestamp ts(1700000000000L, 123456); - Literal lit(ts); - auto predicate = PredicateBuilder::Equal(0, "ts", FieldType::TIMESTAMP, lit); - - auto arrow_type = arrow::timestamp(arrow::TimeUnit::MICRO); - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"ts"}, {arrow_type}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); +TEST_F(BucketSelectConverterTest, UnsupportedFieldTypeReturnsError) { + auto predicate = + PredicateBuilder::Equal(0, "items", FieldType::ARRAY, Literal(static_cast(42))); - // Verify: precision=6 uses non-compact WriteTimestamp (different layout than precision=3) - auto row = BinaryRowGenerator::GenerateRow({TimestampType(ts, 6)}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); + Result> result = + BucketSelectConverter::Convert(predicate, {"items"}, {arrow::list(arrow::int32())}, + BucketFunctionType::DEFAULT, 5, pool_.get()); + ASSERT_NOK_WITH_MSG(result.status(), "unsupported field type"); } -TEST_F(BucketSelectConverterTest, DecimalKey) { - int32_t num_buckets = 10; - Decimal dec = Decimal::FromUnscaledLong(12345L, 10, 2); - Literal lit(dec); - auto predicate = PredicateBuilder::Equal(0, "amount", FieldType::DECIMAL, lit); - - auto arrow_type = arrow::decimal128(10, 2); - ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert( - predicate, {"amount"}, {arrow_type}, - BucketFunctionType::DEFAULT, num_buckets, pool_.get())); - ASSERT_TRUE(result.has_value()); - - // Verify - auto row = BinaryRowGenerator::GenerateRow({dec}, pool_.get()); - DefaultBucketFunction func; - ASSERT_EQ(func.Bucket(row, num_buckets), result.value()); +TEST_F(BucketSelectConverterTest, ModBucketFunctionWithMultipleKeysReturnsError) { + auto id_predicate = + PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(42))); + auto region_predicate = + PredicateBuilder::Equal(1, "region", FieldType::INT, Literal(static_cast(1))); + ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({id_predicate, region_predicate})); + + Result> result = BucketSelectConverter::Convert( + predicate, {"id", "region"}, {arrow::int32(), arrow::int32()}, BucketFunctionType::MOD, 5, + pool_.get()); + ASSERT_NOK_WITH_MSG(result.status(), + "MOD bucket function requires exactly one bucket key field"); } } // namespace paimon::test diff --git a/src/paimon/core/bucket/default_bucket_function_test.cpp b/src/paimon/core/bucket/default_bucket_function_test.cpp index 16d18ca3..c02eadab 100644 --- a/src/paimon/core/bucket/default_bucket_function_test.cpp +++ b/src/paimon/core/bucket/default_bucket_function_test.cpp @@ -18,14 +18,28 @@ #include "paimon/core/bucket/default_bucket_function.h" +#include + #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +void CheckDefaultBucket(const DefaultBucketFunction& func, const BinaryRow& row, + int32_t expected_hash, int32_t expected_bucket) { + constexpr int32_t kNumBuckets = 1000; + ASSERT_EQ(expected_hash, row.HashCode()); + ASSERT_EQ(expected_bucket, func.Bucket(row, kNumBuckets)); +} + +} // namespace + TEST(DefaultBucketFunctionTest, TestBasicHashMod) { auto pool = GetDefaultPool(); DefaultBucketFunction func; @@ -81,4 +95,45 @@ TEST(DefaultBucketFunctionTest, TestMultiFieldRow) { ASSERT_EQ(std::abs(row.HashCode() % num_buckets), bucket); } +TEST(DefaultBucketFunctionTest, TestFloatSpecialValuesCompatibleWithJava) { + auto pool = GetDefaultPool(); + DefaultBucketFunction func; + + // Verified with Java DefaultBucketFunction and NUM_BUCKETS = 1000. + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::quiet_NaN()}, pool.get()), + -2039172089, 89); + CheckDefaultBucket( + func, BinaryRowGenerator::GenerateRow({std::numeric_limits::infinity()}, pool.get()), + 2139216202, 202); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({-std::numeric_limits::infinity()}, pool.get()), + -106221671, 671); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({0.0f}, pool.get()), -300363099, 99); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({-0.0f}, pool.get()), 916225219, 219); +} + +TEST(DefaultBucketFunctionTest, TestDoubleSpecialValuesCompatibleWithJava) { + auto pool = GetDefaultPool(); + DefaultBucketFunction func; + + // Verified with Java DefaultBucketFunction and NUM_BUCKETS = 1000. + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::quiet_NaN()}, pool.get()), + -1323214697, 697); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::infinity()}, pool.get()), + -1556713404, 404); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({-std::numeric_limits::infinity()}, pool.get()), + -2079171840, 840); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({0.0}, pool.get()), -300363099, 99); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({-0.0}, pool.get()), 302122119, 119); +} + } // namespace paimon::test diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp index 72f0f006..913053c1 100644 --- a/src/paimon/core/bucket/hive_bucket_function.cpp +++ b/src/paimon/core/bucket/hive_bucket_function.cpp @@ -71,15 +71,15 @@ Result> HiveBucketFunction::Create( } int32_t HiveBucketFunction::Bucket(const BinaryRow& row, int32_t num_buckets) const { - static constexpr int32_t SEED = 0; - int32_t hash = SEED; + static constexpr uint32_t SEED = 0; + uint32_t hash = SEED; for (int32_t i = 0; i < row.GetFieldCount(); i++) { - hash = (31 * hash) + ComputeHash(row, i); + hash = 31U * hash + ComputeHash(row, i); } - return Mod(hash & std::numeric_limits::max(), num_buckets); + return Mod(static_cast(hash & 0x7FFFFFFF), num_buckets); } -int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_index) const { +uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_index) const { if (row.IsNullAt(field_index)) { return 0; } @@ -88,20 +88,25 @@ int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_inde switch (info.type) { case FieldType::BOOLEAN: return HiveHasher::HashInt(row.GetBoolean(field_index) ? 1 : 0); - case FieldType::TINYINT: - return HiveHasher::HashInt(static_cast(row.GetByte(field_index))); + case FieldType::TINYINT: { + auto byte = static_cast(row.GetByte(field_index)); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + return HiveHasher::HashInt(static_cast(signed_byte)); + } case FieldType::SMALLINT: - return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); + return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); case FieldType::INT: case FieldType::DATE: - return HiveHasher::HashInt(row.GetInt(field_index)); + return HiveHasher::HashInt(static_cast(row.GetInt(field_index))); case FieldType::BIGINT: - return HiveHasher::HashLong(row.GetLong(field_index)); + return HiveHasher::HashLong(static_cast(row.GetLong(field_index))); case FieldType::FLOAT: { float float_value = row.GetFloat(field_index); - int32_t bits; + uint32_t bits; if (float_value == -0.0f) { bits = 0; + } else if (std::isnan(float_value)) { + bits = 0x7FC00000U; } else { std::memcpy(&bits, &float_value, sizeof(bits)); } @@ -109,9 +114,11 @@ int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_inde } case FieldType::DOUBLE: { double double_value = row.GetDouble(field_index); - int64_t bits; + uint64_t bits; if (double_value == -0.0) { - bits = 0L; + bits = 0; + } else if (std::isnan(double_value)) { + bits = 0x7FF8000000000000ULL; } else { std::memcpy(&bits, &double_value, sizeof(bits)); } diff --git a/src/paimon/core/bucket/hive_bucket_function.h b/src/paimon/core/bucket/hive_bucket_function.h index 82f0c40d..7b742634 100644 --- a/src/paimon/core/bucket/hive_bucket_function.h +++ b/src/paimon/core/bucket/hive_bucket_function.h @@ -55,7 +55,7 @@ class HiveBucketFunction : public BucketFunction { explicit HiveBucketFunction(const std::vector& field_infos); /// Compute the Hive hash for a single field value. - int32_t ComputeHash(const BinaryRow& row, int32_t field_index) const; + uint32_t ComputeHash(const BinaryRow& row, int32_t field_index) const; /// Mod operation that always returns non-negative result. static int32_t Mod(int32_t value, int32_t divisor); diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp index 73a94c5a..21f2a984 100644 --- a/src/paimon/core/bucket/hive_bucket_function_test.cpp +++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp @@ -18,6 +18,7 @@ #include "paimon/core/bucket/hive_bucket_function.h" +#include #include #include "gtest/gtest.h" @@ -100,6 +101,28 @@ class HiveBucketFunctionTest : public ::testing::Test { auto pool = GetDefaultPool(); return BinaryRowGenerator::GenerateRow({value}, pool.get()); } + + BinaryRow CreateByteRow(int8_t value) { + auto pool = GetDefaultPool(); + return BinaryRowGenerator::GenerateRow({value}, pool.get()); + } + + BinaryRow CreateShortRow(int16_t value) { + auto pool = GetDefaultPool(); + return BinaryRowGenerator::GenerateRow({value}, pool.get()); + } + + float FloatFromBits(uint32_t bits) { + float value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } + + double DoubleFromBits(uint64_t bits) { + double value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } }; /// Test matching Java: testHiveBucketFunction @@ -119,12 +142,12 @@ TEST_F(HiveBucketFunctionTest, TestHiveBucketFunction) { // Verify individual hash components: // HiveHasher.hashBytes("hello") = 99162322 - ASSERT_EQ(99162322, HiveHasher::HashBytes("hello", 5)); + ASSERT_EQ(99162322U, HiveHasher::HashBytes("hello", 5)); // HiveHasher.hashBytes({1,2,3}) = 1026 - ASSERT_EQ(1026, HiveHasher::HashBytes("\x01\x02\x03", 3)); + ASSERT_EQ(1026U, HiveHasher::HashBytes("\x01\x02\x03", 3)); // BigDecimal("12.34").hashCode() = 1234 * 31 + 2 = 38256 // (After normalizing "12.3400" -> "12.34", unscaled=1234, scale=2) - ASSERT_EQ(38256, HiveHasher::HashDecimal(Decimal::FromUnscaledLong(123400, 10, 4))); + ASSERT_EQ(38256U, HiveHasher::HashDecimal(Decimal::FromUnscaledLong(123400, 10, 4))); // expectedHash = 31*(31*(31*7 + 99162322) + 1026) + 38256 = 805989529 (with int32 overflow) // bucket = (805989529 & INT32_MAX) % 8 = 1 @@ -207,6 +230,47 @@ TEST_F(HiveBucketFunctionTest, TestDoubleNegativeZero) { ASSERT_EQ(func->Bucket(CreateDoubleRow(0.0), 5), func->Bucket(CreateDoubleRow(-0.0), 5)); } +TEST_F(HiveBucketFunctionTest, TestFloatNaNCanonicalizationCompatibleWithJava) { + std::vector field_types = {FieldType::FLOAT}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction: + // Float.NaN, Float.intBitsToFloat(0x7fa12345), and Float.intBitsToFloat(0x7fc00000) + // all hash through Float.floatToIntBits(...) = 0x7fc00000. + ASSERT_EQ(344, func->Bucket(CreateFloatRow(std::numeric_limits::quiet_NaN()), 1000)); + ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FA12345U)), 1000)); + ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FC00000U)), 1000)); +} + +TEST_F(HiveBucketFunctionTest, TestDoubleNaNCanonicalizationCompatibleWithJava) { + std::vector field_types = {FieldType::DOUBLE}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction: + // Double.NaN, Double.longBitsToDouble(0x7ff123456789abcd), and canonical NaN + // all hash through Double.doubleToLongBits(...) = 0x7ff8000000000000. + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(std::numeric_limits::quiet_NaN()), 1000)); + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF123456789ABCDULL)), 1000)); + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF8000000000000ULL)), 1000)); +} + +TEST_F(HiveBucketFunctionTest, TestTinyintNegativeValuesCompatibleWithJava) { + std::vector field_types = {FieldType::TINYINT}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction using DataTypes.TINYINT(). + ASSERT_EQ(647, func->Bucket(CreateByteRow(static_cast(-1)), 1000)); + ASSERT_EQ(520, func->Bucket(CreateByteRow(std::numeric_limits::min()), 1000)); +} + +TEST_F(HiveBucketFunctionTest, TestSmallintField) { + std::vector field_types = {FieldType::SMALLINT}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + ASSERT_EQ(234, func->Bucket(CreateShortRow(static_cast(1234)), 1000)); + ASSERT_EQ(647, func->Bucket(CreateShortRow(static_cast(-1)), 1000)); +} + /// Test STRING field TEST_F(HiveBucketFunctionTest, TestStringField) { std::vector field_types = {FieldType::STRING}; diff --git a/src/paimon/core/bucket/hive_hasher.h b/src/paimon/core/bucket/hive_hasher.h index d86be406..e9dad0e4 100644 --- a/src/paimon/core/bucket/hive_hasher.h +++ b/src/paimon/core/bucket/hive_hasher.h @@ -30,21 +30,23 @@ namespace paimon { /// hash implementation, ensuring consistent bucket assignment between Paimon C++ and Java. class HiveHasher { public: - /// Hash an int value (identity function, same as Hive). - static int32_t HashInt(int32_t input) { + /// Hash an int value (identity function, same as Hive's 32-bit int hash). + static uint32_t HashInt(uint32_t input) { return input; } /// Hash a long value (same as Java's Long.hashCode). - static int32_t HashLong(int64_t input) { - return static_cast(input ^ (static_cast(input) >> 32)); + static uint32_t HashLong(uint64_t input) { + return static_cast(input ^ (input >> 32)); } /// Hash a byte array. - static int32_t HashBytes(const char* bytes, int32_t length) { - int32_t result = 0; + static uint32_t HashBytes(const char* bytes, int32_t length) { + uint32_t result = 0; for (int32_t i = 0; i < length; i++) { - result = (result * 31) + static_cast(bytes[i]); + auto byte = static_cast(bytes[i]); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + result = result * 31U + static_cast(signed_byte); } return result; } @@ -60,7 +62,7 @@ class HiveHasher { /// /// @param decimal The decimal value to normalize. /// @return The hash code of the normalized decimal, computed as Java BigDecimal.hashCode(). - static int32_t HashDecimal(const Decimal& decimal) { + static uint32_t HashDecimal(const Decimal& decimal) { // Java BigDecimal.hashCode() = unscaledValue.intValue() * 31 + scale // For compact decimals (precision <= 18), we can use the long value directly. // For non-compact decimals, we need to handle the 128-bit value. @@ -92,7 +94,9 @@ class HiveHasher { } // Count integer digits - auto abs_value = value < 0 ? -value : value; + auto abs_value = + value < 0 ? static_cast(0) - static_cast(value) + : static_cast(value); int32_t total_digits = 0; auto temp = abs_value; while (temp > 0) { @@ -148,8 +152,8 @@ class HiveHasher { // Compute Java BigDecimal.hashCode(): // hashCode = intValue(unscaledValue) * 31 + scale // intValue() returns the low 32 bits of the value - auto int_value = static_cast(static_cast(value)); - return int_value * 31 + scale; + auto int_value = static_cast(value); + return int_value * 31U + static_cast(scale); } private: diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp new file mode 100644 index 00000000..5642e62b --- /dev/null +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/binary_to_blob_cast_executor.h" + +#include +#include + +#include "arrow/array/array_binary.h" +#include "arrow/buffer.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/status.h" + +namespace arrow { +class Array; +} // namespace arrow + +namespace paimon { +Result BinaryToBlobCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor does not support literal cast from {} to {}", + static_cast(literal.GetType()), target_type->ToString())); +} + +Result> BinaryToBlobCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (array->type_id() != arrow::Type::BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports binary input, got {}", + array->type()->ToString())); + } + if (target_type->id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports large_binary target, got {}", + target_type->ToString())); + } + + auto binary_array = std::static_pointer_cast(array); + if (binary_array->offset() != 0) { + return Status::Invalid("BinaryToBlobCastExecutor only supports arrays with zero offset"); + } + + const int64_t length = binary_array->length(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr large_offsets_buffer, + arrow::AllocateBuffer((length + 1) * static_cast(sizeof(int64_t)), pool)); + auto* large_offsets = reinterpret_cast(large_offsets_buffer->mutable_data()); + for (int64_t row_index = 0; row_index <= length; row_index++) { + large_offsets[row_index] = binary_array->value_offset(row_index); + } + + std::shared_ptr null_bitmap = binary_array->null_bitmap(); + if (binary_array->null_count() == 0) { + null_bitmap.reset(); + } + + auto value_data = binary_array->value_data(); + auto array_data = + arrow::ArrayData::Make(target_type, length, {null_bitmap, large_offsets_buffer, value_data}, + binary_array->null_count()); + return arrow::MakeArray(array_data); +} + +} // namespace paimon diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.h b/src/paimon/core/casting/binary_to_blob_cast_executor.h new file mode 100644 index 00000000..8ea06a3a --- /dev/null +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +class BinaryToBlobCastExecutor : public CastExecutor { + public: + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/casting/cast_executor_factory.cpp b/src/paimon/core/casting/cast_executor_factory.cpp index 2c0a4116..8e56ca04 100644 --- a/src/paimon/core/casting/cast_executor_factory.cpp +++ b/src/paimon/core/casting/cast_executor_factory.cpp @@ -21,6 +21,7 @@ #include +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -149,6 +150,8 @@ CastExecutorFactory::CastExecutorFactory() { REGISTER_CAST_EXECUTOR(STRING, BINARY, BinaryToStringCastExecutor); + REGISTER_CAST_EXECUTOR(BLOB, BINARY, BinaryToBlobCastExecutor); + REGISTER_CAST_EXECUTOR(STRING, DATE, DateToStringCastExecutor); REGISTER_CAST_EXECUTOR(TIMESTAMP, DATE, DateToTimestampCastExecutor); diff --git a/src/paimon/core/casting/cast_executor_factory_test.cpp b/src/paimon/core/casting/cast_executor_factory_test.cpp index 56acbe48..9ab7d168 100644 --- a/src/paimon/core/casting/cast_executor_factory_test.cpp +++ b/src/paimon/core/casting/cast_executor_factory_test.cpp @@ -20,6 +20,7 @@ #include "paimon/core/casting/cast_executor_factory.h" #include "gtest/gtest.h" +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -123,6 +124,13 @@ TEST(CastExecutorFactoryTest, TestRegister) { ASSERT_TRUE(cast_executor); ASSERT_TRUE(std::dynamic_pointer_cast(cast_executor)); } + { + auto* factory = CastExecutorFactory::GetCastExecutorFactory(); + ASSERT_FALSE(factory->executor_map_.empty()); + auto cast_executor = factory->GetCastExecutor(FieldType::BINARY, FieldType::BLOB); + ASSERT_TRUE(cast_executor); + ASSERT_TRUE(std::dynamic_pointer_cast(cast_executor)); + } { auto* factory = CastExecutorFactory::GetCastExecutorFactory(); ASSERT_FALSE(factory->executor_map_.empty()); diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index 7e122549..ed540ae4 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -1302,6 +1303,43 @@ TEST_F(CastExecutorTest, TestBinaryToStringCastExecutorCastArray) { } } +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastLiteral) { + auto cast_executor = std::make_shared(); + std::string src_data = "blob-descriptor-bytes"; + ASSERT_NOK_WITH_MSG( + cast_executor->Cast(Literal(FieldType::BINARY, src_data.data(), src_data.size()), + arrow::large_binary()), + "BinaryToBlobCastExecutor does not support literal cast"); +} + +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArray) { + auto cast_executor = std::make_shared(); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::binary(), R"(["foo", "bar", "", null, "blob"])") + .ValueOrDie(); + auto expected_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::large_binary(), R"(["foo", "bar", "", null, "blob"])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr target_array, + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool())); + ASSERT_TRUE(target_array->Equals(expected_array)); + ASSERT_EQ(target_array->data()->buffers[2], src_array->data()->buffers[2]); +} + +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArrayWithOffset) { + auto cast_executor = std::make_shared(); + auto src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["skip", "foo", "bar"])") + .ValueOrDie() + ->Slice(1, 2); + + ASSERT_NOK_WITH_MSG( + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool()), + "BinaryToBlobCastExecutor only supports arrays with zero offset"); +} + TEST_F(CastExecutorTest, TestDateToStringCastExecutorCastLiteral) { auto cast_executor = std::make_shared(); // date values ranging from 0000-01-01 to 9999-12-31 @@ -2487,7 +2525,7 @@ TEST_F(CastExecutorTest, TestBooleanToDecimalCastExecutorCastLiteral) { } { ASSERT_OK_AND_ASSIGN(Literal valid_literal, - cast_executor->Cast(Literal(false), arrow::decimal128(3, 3))) + cast_executor->Cast(Literal(false), arrow::decimal128(3, 3))); ASSERT_EQ(valid_literal, Literal(Decimal(3, 3, 0))); } } diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 4c1b06bb..ab26ce0c 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -34,7 +34,7 @@ Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); - return std::make_unique(core_options.GetFileSystem(), root_path); + return std::make_unique(core_options.GetFileSystem(), root_path, options); } } // namespace paimon diff --git a/src/paimon/core/catalog/commit_table_request_test.cpp b/src/paimon/core/catalog/commit_table_request_test.cpp index 39b9ff95..deafee36 100644 --- a/src/paimon/core/catalog/commit_table_request_test.cpp +++ b/src/paimon/core/catalog/commit_table_request_test.cpp @@ -39,7 +39,7 @@ TEST(CommitTableRequestTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); std::vector partition_statistics = { @@ -58,12 +58,10 @@ TEST(CommitTableRequestTest, TestSimple) { "baseManifestListSize": 291, "deltaManifestList": "manifest-list-3879e56f-2f27-49ae-a2f3-3dcbb8eb0beb-1", "deltaManifestListSize": 1342, - "changelogManifestList": null, "commitUser": "commit_user_1", "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", "timeMillis": 1758097357597, - "logOffsets": {}, "totalRecordCount": 5, "deltaRecordCount": 5, "changelogRecordCount": 0, diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 277f0b8a..292af2e8 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" @@ -49,8 +50,12 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, - const std::string& warehouse) - : fs_(fs), warehouse_(warehouse), logger_(Logger::GetLogger("FileSystemCatalog")) {} + const std::string& warehouse, + const std::map& catalog_options) + : fs_(fs), + warehouse_(warehouse), + catalog_options_(catalog_options), + logger_(Logger::GetLogger("FileSystemCatalog")) {} Status FileSystemCatalog::CreateDatabase(const std::string& db_name, const std::map& options, @@ -90,13 +95,16 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented( - "do not support checking DatabaseExists for system database."); + return true; } return fs_->Exists(NewDatabasePath(warehouse_, db_name)); } Result FileSystemCatalog::TableExists(const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, @@ -119,7 +127,7 @@ std::string FileSystemCatalog::GetDatabaseLocation(const std::string& db_name) c return NewDatabasePath(warehouse_, db_name); } -std::string FileSystemCatalog::GetTableLocation(const Identifier& identifier) const { +Result FileSystemCatalog::GetTableLocation(const Identifier& identifier) const { return NewDataTablePath(warehouse_, identifier); } @@ -158,7 +166,8 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* return Status::NotImplemented( "create table operation does not support object store file system for now"); } - SchemaManager schema_manager(fs_, NewDataTablePath(warehouse_, identifier)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, NewDataTablePath(warehouse_, identifier)); + SchemaManager schema_manager(fs_, table_path); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_schema, schema_manager.CreateTable(schema, partition_keys, primary_keys, options)); @@ -172,7 +181,8 @@ Result>> FileSystemCatalog::TableSche return Status::NotImplemented( "do not support checking TableSchemaExists for system table."); } - SchemaManager schema_manager(fs_, NewDataTablePath(warehouse_, identifier)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, NewDataTablePath(warehouse_, identifier)); + SchemaManager schema_manager(fs_, table_path); return schema_manager.Latest(); } @@ -184,6 +194,10 @@ std::shared_ptr FileSystemCatalog::GetFileSystem() const { return fs_; } +const std::map& FileSystemCatalog::GetOptions() const { + return catalog_options_; +} + bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { return db_name == SYSTEM_DATABASE_NAME; } @@ -204,10 +218,11 @@ std::string FileSystemCatalog::NewDatabasePath(const std::string& warehouse, return PathUtil::JoinPath(warehouse, db_name + DB_SUFFIX); } -std::string FileSystemCatalog::NewDataTablePath(const std::string& warehouse, - const Identifier& identifier) { +Result FileSystemCatalog::NewDataTablePath(const std::string& warehouse, + const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); return PathUtil::JoinPath(NewDatabasePath(warehouse, identifier.GetDatabaseName()), - identifier.GetTableName()); + data_table_name); } Result> FileSystemCatalog::ListDatabases() const { @@ -227,7 +242,7 @@ Result> FileSystemCatalog::ListDatabases() const { Result> FileSystemCatalog::ListTables(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented("do not support listing tables for system database."); + return GlobalSystemTableLoader::GetSupportedTableNames(catalog_options_); } std::string database_path = NewDatabasePath(warehouse_, db_name); std::vector> file_status_list; @@ -260,6 +275,24 @@ Result FileSystemCatalog::TableExistsInFileSystem(const std::string& table Result> FileSystemCatalog::LoadTableSchema( const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetTableName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, @@ -279,9 +312,9 @@ Result> FileSystemCatalog::LoadTableSchema( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, - GetTableLocation(data_identifier), + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); @@ -302,7 +335,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide return std::make_shared(schema, identifier.GetDatabaseName(), identifier.GetTableName()); } - return Table::Create(fs_, GetTableLocation(identifier), identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + return Table::Create(fs_, table_path, identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -367,33 +401,17 @@ Result> FileSystemCatalog::GetSchemaExternalPaths( Result> FileSystemCatalog::GetTableBranches( const std::string& table_path) const { - std::vector branches; - std::string branch_dir = PathUtil::JoinPath(table_path, "branch"); - PAIMON_ASSIGN_OR_RAISE(bool branch_dir_exists, fs_->Exists(branch_dir)); - if (!branch_dir_exists) { - return branches; - } - - std::vector> file_status_list; - PAIMON_RETURN_NOT_OK(fs_->ListDir(branch_dir, &file_status_list)); - - for (const auto& file_status : file_status_list) { - if (file_status->IsDir()) { - std::string dir_name = PathUtil::GetName(file_status->GetPath()); - // Branch directory name format: branch-{branch_name} - const std::string branch_prefix = BranchManager::BRANCH_PREFIX; - if (StringUtils::StartsWith(dir_name, branch_prefix, /*start_pos=*/0)) { - std::string branch_name = dir_name.substr(branch_prefix.length()); - branches.push_back(branch_name); - } - } - } + PAIMON_ASSIGN_OR_RAISE(std::vector branches, + BranchManager::ListBranches(fs_, table_path)); + branches.erase( + std::remove(branches.begin(), branches.end(), BranchManager::DEFAULT_MAIN_BRANCH), + branches.end()); return branches; } Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, const std::vector& external_paths) { - std::string table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // Delete external paths first for (const auto& external_path : external_paths) { @@ -413,8 +431,7 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if if (is_system_table) { return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); } - - std::string table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { if (ignore_if_not_exists) { @@ -500,8 +517,8 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::Invalid(fmt::format("target table {} already exists", to_table.ToString())); } - std::string from_path = GetTableLocation(from_table); - std::string to_path = GetTableLocation(to_table); + PAIMON_ASSIGN_OR_RAISE(std::string from_path, GetTableLocation(from_table)); + PAIMON_ASSIGN_OR_RAISE(std::string to_path, GetTableLocation(to_table)); PAIMON_RETURN_NOT_OK(fs_->Rename(from_path, to_path)); return Status::OK(); } @@ -530,8 +547,7 @@ Result> FileSystemCatalog::ListSnapshots( if (!exists) { return Status::NotExist(fmt::format("table {} does not exist", identifier.ToString())); } - - auto table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); SnapshotManager mgr(fs_, table_path, branch); PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, mgr.GetAllSnapshots()); std::sort(snapshots.begin(), snapshots.end(), diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index c4f83064..3a464ae8 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -40,7 +40,8 @@ class Logger; class FileSystemCatalog : public Catalog { public: - FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse); + FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, + const std::map& catalog_options); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -59,17 +60,19 @@ class FileSystemCatalog : public Catalog { Result DatabaseExists(const std::string& db_name) const override; Result TableExists(const Identifier& identifier) const override; std::string GetDatabaseLocation(const std::string& db_name) const override; - std::string GetTableLocation(const Identifier& identifier) const override; + Result GetTableLocation(const Identifier& identifier) const override; Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; + const std::map& GetOptions() const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; private: static std::string NewDatabasePath(const std::string& warehouse, const std::string& db_name); - static std::string NewDataTablePath(const std::string& warehouse, const Identifier& identifier); + static Result NewDataTablePath(const std::string& warehouse, + const Identifier& identifier); static bool IsSystemDatabase(const std::string& db_name); static Result IsSpecifiedSystemTable(const Identifier& identifier); static Result IsSystemTable(const Identifier& identifier); @@ -93,6 +96,7 @@ class FileSystemCatalog : public Catalog { std::shared_ptr fs_; std::string warehouse_; + std::map catalog_options_; std::shared_ptr logger_; }; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 0ab2ac27..2a07b262 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -18,6 +18,8 @@ #include "paimon/core/catalog/file_system_catalog.h" +#include + #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" @@ -28,6 +30,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" @@ -44,7 +47,7 @@ TEST(FileSystemCatalogTest, TestDatabaseExists) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK_AND_ASSIGN(auto exist, catalog.DatabaseExists("db1")); ASSERT_FALSE(exist); @@ -70,7 +73,7 @@ TEST(FileSystemCatalogTest, TestInvalidCreateDatabase) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG( catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true), @@ -87,12 +90,12 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG(catalog.CreateDatabase(Catalog::SYSTEM_DATABASE_NAME, options, /*ignore_if_exists=*/true), "Cannot create database for system database"); } - // do not support create system table + /// Do not support create system table. { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -100,7 +103,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), @@ -125,7 +128,7 @@ TEST(FileSystemCatalogTest, TestCreateTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { @@ -161,7 +164,7 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); @@ -179,8 +182,8 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(exists); ASSERT_OK_AND_ASSIGN(exists, catalog.TableExists(Identifier("db1", "missing$options"))); ASSERT_FALSE(exists); - ASSERT_EQ(catalog.GetTableLocation(options_identifier), - PathUtil::JoinPath(PathUtil::JoinPath(dir->Str(), "db1.db"), "tbl1$options")); + ASSERT_OK_AND_ASSIGN(auto table_path, catalog.GetTableLocation(options_identifier)); + ASSERT_EQ(table_path, PathUtil::JoinPath(PathUtil::JoinPath(dir->Str(), "db1.db"), "tbl1")); ASSERT_OK_AND_ASSIGN(std::shared_ptr system_schema, catalog.LoadTableSchema(options_identifier)); @@ -219,7 +222,7 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); auto typed_schema = @@ -283,6 +286,149 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { "Cannot rename system table"); } +TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = + arrow::schema({arrow::field("pk", arrow::utf8()), arrow::field("v", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + std::vector metadata_tables = {"snapshots", "schemas", "tags", "branches", + "consumers", "manifests", "files"}; + for (const auto& table_name : metadata_tables) { + Identifier system_identifier("db1", "tbl1$" + table_name); + ASSERT_OK_AND_ASSIGN(bool exists, catalog.TableExists(system_identifier)); + ASSERT_TRUE(exists) << table_name; + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_schema, + catalog.LoadTableSchema(system_identifier)); + ASSERT_TRUE(std::dynamic_pointer_cast(system_schema) != nullptr) + << table_name; + ASSERT_OK_AND_ASSIGN(auto c_schema, system_schema->GetArrowSchema()); + auto loaded_schema_result = arrow::ImportSchema(c_schema.get()); + ASSERT_TRUE(loaded_schema_result.ok()) << loaded_schema_result.status().ToString(); + ASSERT_GT(loaded_schema_result.ValueUnsafe()->num_fields(), 0) << table_name; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr snapshots_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$snapshots"))); + ASSERT_OK_AND_ASSIGN(auto snapshots_c_schema, snapshots_schema->GetArrowSchema()); + auto snapshots_arrow_schema = arrow::ImportSchema(snapshots_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(snapshots_arrow_schema->field_names(), + (std::vector{ + "snapshot_id", "schema_id", "commit_user", "commit_identifier", "commit_kind", + "commit_time", "base_manifest_list", "delta_manifest_list", + "changelog_manifest_list", "total_record_count", "delta_record_count", + "changelog_record_count", "watermark", "next_row_id"})); + ASSERT_EQ(snapshots_arrow_schema->field(5)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schemas_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$schemas"))); + ASSERT_OK_AND_ASSIGN(auto schemas_c_schema, schemas_schema->GetArrowSchema()); + auto schemas_arrow_schema = arrow::ImportSchema(schemas_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(schemas_arrow_schema->field_names(), + (std::vector{"schema_id", "fields", "partition_keys", "primary_keys", + "options", "comment", "update_time"})); + ASSERT_EQ(schemas_arrow_schema->field(6)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr tags_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$tags"))); + ASSERT_OK_AND_ASSIGN(auto tags_c_schema, tags_schema->GetArrowSchema()); + auto tags_arrow_schema = arrow::ImportSchema(tags_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(tags_arrow_schema->field_names(), + (std::vector{"tag_name", "snapshot_id", "schema_id", "commit_time", + "record_count", "create_time", "time_retained"})); + ASSERT_EQ(tags_arrow_schema->field(3)->type()->id(), arrow::Type::TIMESTAMP); + ASSERT_EQ(tags_arrow_schema->field(5)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr branches_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$branches"))); + ASSERT_OK_AND_ASSIGN(auto branches_c_schema, branches_schema->GetArrowSchema()); + auto branches_arrow_schema = arrow::ImportSchema(branches_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(branches_arrow_schema->field_names(), + (std::vector{"branch_name", "create_time"})); + ASSERT_EQ(branches_arrow_schema->field(1)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr consumers_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$consumers"))); + ASSERT_OK_AND_ASSIGN(auto consumers_c_schema, consumers_schema->GetArrowSchema()); + auto consumers_arrow_schema = arrow::ImportSchema(consumers_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(consumers_arrow_schema->field_names(), + (std::vector{"consumer_id", "next_snapshot_id"})); + ASSERT_FALSE(consumers_arrow_schema->field(1)->nullable()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr manifests_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$manifests"))); + ASSERT_OK_AND_ASSIGN(auto manifests_c_schema, manifests_schema->GetArrowSchema()); + auto manifests_arrow_schema = arrow::ImportSchema(manifests_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(manifests_arrow_schema->field_names(), + (std::vector{"file_name", "file_size", "num_added_files", + "num_deleted_files", "schema_id", "min_partition_stats", + "max_partition_stats", "min_row_id", "max_row_id"})); + ASSERT_FALSE(manifests_arrow_schema->field(0)->nullable()); + ASSERT_EQ(manifests_arrow_schema->field(1)->type()->id(), arrow::Type::INT64); + ASSERT_FALSE(manifests_arrow_schema->field(4)->nullable()); + ASSERT_TRUE(manifests_arrow_schema->field(5)->nullable()); + ASSERT_TRUE(manifests_arrow_schema->field(8)->nullable()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr files_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$files"))); + ASSERT_OK_AND_ASSIGN(auto files_c_schema, files_schema->GetArrowSchema()); + auto files_arrow_schema = arrow::ImportSchema(files_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(files_arrow_schema->field_names(), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_TRUE(files_arrow_schema->field(0)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(1)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(2)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(10)->nullable()); + ASSERT_EQ(files_arrow_schema->field(15)->type()->id(), arrow::Type::TIMESTAMP); + ASSERT_EQ(files_arrow_schema->field(19)->type()->id(), arrow::Type::LIST); + auto write_cols_type = + std::dynamic_pointer_cast(files_arrow_schema->field(19)->type()); + ASSERT_TRUE(write_cols_type); + ASSERT_EQ(write_cols_type->value_type()->id(), arrow::Type::STRING); + + Identifier snapshots_identifier("db1", "tbl1$snapshots"); + ::ArrowSchema system_create_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); + ASSERT_NOK_WITH_MSG( + catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), + "Cannot create table for system table"); + ArrowSchemaRelease(&system_create_schema); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), + "Cannot rename system table"); +} + TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -292,7 +438,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = {arrow::field("f0", arrow::boolean()), @@ -348,7 +494,7 @@ TEST(FileSystemCatalogTest, TestInvalidCreateTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), @@ -373,7 +519,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileDbNotExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), @@ -396,7 +542,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileTableExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); { ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { @@ -450,23 +596,36 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileTableExist) { ASSERT_OK(catalog.CreateTable(identifier, &schema, {"f1"}, {}, options, /*ignore_if_exists=*/true)); ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", dir->Str(), {})); - ASSERT_OK(fs->Delete( - PathUtil::JoinPath(catalog.GetTableLocation(identifier), "schema/schema-0"))); + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog.GetTableLocation(identifier)); + ASSERT_OK(fs->Delete(PathUtil::JoinPath(table_path, "schema/schema-0"))); ASSERT_OK(catalog.CreateTable(identifier, &schema, {"f1"}, {}, options, /*ignore_if_exists=*/false)); } } -TEST(FileSystemCatalogTest, TestInvalidList) { +TEST(FileSystemCatalogTest, TestSystemList) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); - ASSERT_NOK_WITH_MSG(catalog.ListTables("sys"), - "do not support listing tables for system database."); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK_AND_ASSIGN(auto sys_tables, catalog.ListTables("sys")); + ASSERT_FALSE(sys_tables.empty()); + // catalog_options is disabled by default, matching Java Paimon. + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "catalog_options") == + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "all_table_options") != + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "partitions") != sys_tables.end()); + + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + FileSystemCatalog enabled_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK_AND_ASSIGN(auto enabled_sys_tables, enabled_catalog.ListTables("sys")); + ASSERT_TRUE(std::find(enabled_sys_tables.begin(), enabled_sys_tables.end(), + "catalog_options") != enabled_sys_tables.end()); } TEST(FileSystemCatalogTest, TestValidateTableSchema) { @@ -476,7 +635,7 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), @@ -518,8 +677,8 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { ASSERT_NOK(table_schema->GetFieldType("f4")); ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", dir->Str(), {})); - std::string schema_path = - PathUtil::JoinPath(catalog.GetTableLocation(identifier), "schema/schema-0"); + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog.GetTableLocation(identifier)); + std::string schema_path = PathUtil::JoinPath(table_path, "schema/schema-0"); std::string expected_json_schema; ASSERT_OK(fs->ReadFile(schema_path, &expected_json_schema)); @@ -545,7 +704,7 @@ TEST(FileSystemCatalogTest, TestDropDatabase) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); // Test 1: Drop non-existent database with ignore_if_not_exists=true ASSERT_OK(catalog.DropDatabase("non_existent_db", /*ignore_if_not_exists=*/true, @@ -601,7 +760,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Test 1: Drop non-existent table with ignore_if_not_exists=true @@ -627,7 +786,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_OK_AND_ASSIGN(bool exist, catalog.TableExists(Identifier("test_db", "tbl1"))); ASSERT_FALSE(exist); - // Test 4: Drop system table + /// Test 4: Drop system table. ASSERT_NOK_WITH_MSG( catalog.DropTable(Identifier("test_db", "tbl$system"), /*ignore_if_not_exists=*/false), @@ -643,7 +802,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Test 1: Rename non-existent table with ignore_if_not_exists=true @@ -693,7 +852,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { /*ignore_if_not_exists=*/false), "Cannot rename table across databases. Cross-database rename is not supported."); - // Test 6: Rename system table + /// Test 6: Rename system table. ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), @@ -710,7 +869,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithExternalPath) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create external path directory @@ -772,7 +931,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithMultipleExternalPaths) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create multiple external path directories @@ -829,7 +988,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create external path directory for global index @@ -933,7 +1092,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnBranch) { ASSERT_TRUE(external_exists); // Drop the table via catalog - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier identifier("test_db", "append_table_with_rt_branch"); ASSERT_OK_AND_ASSIGN(bool table_exists, catalog.TableExists(identifier)); ASSERT_TRUE(table_exists); @@ -961,7 +1120,7 @@ TEST(FileSystemCatalogTest, TestListSnapshots) { std::string db_path = dir->Str() + "/test_db.db"; ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, db_path)); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier id("test_db", "append_table_with_multiple_file_format"); ASSERT_OK_AND_ASSIGN(std::vector snapshots, catalog.ListSnapshots(id, "")); @@ -995,7 +1154,7 @@ TEST(FileSystemCatalogTest, TestListSnapshotsTableNotExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG( catalog.ListSnapshots(Identifier("non_existent_db", "non_existent_table"), ""), @@ -1053,7 +1212,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithBranchExternalPaths) { ASSERT_TRUE(external_exists); // Drop the table via catalog - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier identifier("test_db", "append_table_with_rt_branch"); ASSERT_OK_AND_ASSIGN(bool table_exists, catalog.TableExists(identifier)); ASSERT_TRUE(table_exists); diff --git a/src/paimon/core/catalog/identifier.cpp b/src/paimon/core/catalog/identifier.cpp index 3b403d30..32ba446a 100644 --- a/src/paimon/core/catalog/identifier.cpp +++ b/src/paimon/core/catalog/identifier.cpp @@ -23,6 +23,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/murmurhash_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/result.h" #include "paimon/status.h" @@ -40,8 +41,11 @@ Identifier::Identifier(const std::string& table) Identifier::Identifier(const std::string& database, const std::string& table) : database_(database), table_(table) {} -bool Identifier::operator==(const Identifier& other) { - return (database_ == other.database_ && table_ == other.table_); +bool Identifier::operator==(const Identifier& other) const { + if (this == &other) { + return true; + } + return database_ == other.database_ && table_ == other.table_; } const std::string& Identifier::GetDatabaseName() const { @@ -81,6 +85,35 @@ std::string Identifier::ToString() const { return fmt::format("Identifier{{database='{}', table='{}'}}", database_, table_); } +int32_t Identifier::HashCode() const { + int32_t hash = MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(database_.data()), + /*offset=*/0, database_.size()); + return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(table_.data()), + /*offset=*/0, table_.size(), hash); +} + +std::string Identifier::GetFullName() const { + if (database_ == kUnknownDatabase) { + return table_; + } + return fmt::format("{}.{}", database_, table_); +} + +Result Identifier::FromString(const std::string& full_name) { + if (StringUtils::IsNullOrWhitespaceOnly(full_name)) { + return Status::Invalid("full name cannot be empty or whitespace only"); + } + // TODO(lisizhuo.lsz): deal with kUnknownDatabase to be done + const auto dot_pos = full_name.find('.'); + if (dot_pos == std::string::npos || dot_pos == 0 || dot_pos == full_name.size() - 1) { + return Status::Invalid( + fmt::format("cannot get splits from '{}' to get database and table", full_name)); + } + std::string database = full_name.substr(0, dot_pos); + std::string table = full_name.substr(dot_pos + 1); + return Identifier(database, table); +} + Status Identifier::SplitTableName() const { if (parsed_) { return Status::OK(); diff --git a/src/paimon/core/catalog/identifier_test.cpp b/src/paimon/core/catalog/identifier_test.cpp index 771c2f39..50afa6fc 100644 --- a/src/paimon/core/catalog/identifier_test.cpp +++ b/src/paimon/core/catalog/identifier_test.cpp @@ -113,6 +113,47 @@ TEST(IdentifierTest, ParseBranchSystemTable) { ASSERT_TRUE(is_system_table); } +TEST(IdentifierTest, TestGetFullName) { + { + // test unknown database + std::string table = "tbl$branch_dev$options"; + Identifier id(table); + ASSERT_EQ(table, id.GetFullName()); + } + { + // test normal database + std::string db = "database"; + std::string table = "tbl$branch_dev$options"; + Identifier id(db, table); + ASSERT_EQ("database.tbl$branch_dev$options", id.GetFullName()); + } +} + +TEST(IdentifierTest, TestFromString) { + { + // test invalid identifier string + ASSERT_NOK_WITH_MSG(Identifier::FromString(""), + "full name cannot be empty or whitespace only"); + ASSERT_NOK_WITH_MSG(Identifier::FromString(" "), + "full name cannot be empty or whitespace only"); + ASSERT_NOK_WITH_MSG(Identifier::FromString("abcd"), "cannot get splits from 'abcd'"); + ASSERT_NOK_WITH_MSG(Identifier::FromString(".abcd"), "cannot get splits from '.abcd'"); + ASSERT_NOK_WITH_MSG(Identifier::FromString("abcd."), "cannot get splits from 'abcd.'"); + } + { + // test normal database + ASSERT_OK_AND_ASSIGN(Identifier identifier, Identifier::FromString("ab.cd")); + Identifier expected("ab", "cd"); + ASSERT_EQ(identifier, expected); + } + { + // test normal database + ASSERT_OK_AND_ASSIGN(Identifier identifier, Identifier::FromString("ab.cd.ef")); + Identifier expected("ab", "cd.ef"); + ASSERT_EQ(identifier, expected); + } +} + TEST(IdentifierTest, InvalidSystemTableName) { Identifier invalid_middle("db", "tbl$bad$options"); ASSERT_NOK_WITH_MSG(invalid_middle.IsSystemTable(), diff --git a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp index f00da0e6..c415390a 100644 --- a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp +++ b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp @@ -48,7 +48,7 @@ TEST(RenamingSnapshotCommitTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); ASSERT_OK_AND_ASSIGN(bool success, commit->Commit(snapshot, /*statistics=*/{})); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 4cb0c81a..8130dc45 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -28,6 +28,7 @@ #include "paimon/common/fs/resolving_file_system.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/options/time_duration.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/options/expire_config.h" @@ -228,6 +229,23 @@ class ConfigParser { return Status::OK(); } + // Parse VariantShreddingInferenceMode + Status ParseVariantShreddingInferenceMode(VariantShreddingInferenceMode* inference_mode) const { + auto iter = config_map_.find(Options::VARIANT_SHREDDING_INFERENCE_MODE); + if (iter != config_map_.end()) { + std::string str = StringUtils::ToLowerCase(iter->second); + if (str == "per-file") { + *inference_mode = VariantShreddingInferenceMode::PER_FILE; + } else if (str == "adaptive") { + *inference_mode = VariantShreddingInferenceMode::ADAPTIVE; + } else { + return Status::Invalid( + fmt::format("invalid variant shredding inference mode: {}", str)); + } + } + return Status::OK(); + } + // Parse ChangelogProducer Status ParseChangelogProducer(ChangelogProducer* changelog_producer) const { auto iter = config_map_.find(Options::CHANGELOG_PRODUCER); @@ -361,6 +379,7 @@ class ConfigParser { struct CoreOptions::Impl { int64_t page_size = 64 * 1024; std::optional target_file_size; + int64_t target_file_row_num = std::numeric_limits::max(); std::optional blob_target_file_size; int64_t source_split_target_size = 128 * 1024 * 1024; int64_t source_split_open_file_cost = 4 * 1024 * 1024; @@ -369,10 +388,13 @@ struct CoreOptions::Impl { int64_t manifest_full_compaction_file_size = 16 * 1024 * 1024; int64_t write_buffer_size = 256 * 1024 * 1024; int64_t commit_timeout = std::numeric_limits::max(); + int64_t commit_min_retry_wait = 10; + int64_t commit_max_retry_wait = 10 * 1000; std::shared_ptr file_format; std::shared_ptr file_system; std::shared_ptr manifest_file_format; + std::shared_ptr cache; std::optional scan_snapshot_id; std::optional scan_timestamp_millis; @@ -382,7 +404,6 @@ struct CoreOptions::Impl { std::vector blob_fields; std::vector blob_descriptor_fields; std::vector blob_view_fields; - std::vector blob_external_storage_fields; std::string partition_default_name = "__DEFAULT_PARTITION__"; StartupMode startup_mode = StartupMode::Default(); @@ -395,13 +416,14 @@ struct CoreOptions::Impl { std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; - std::optional blob_external_storage_path; + std::optional blob_view_upstream_warehouse; std::map raw_options; int32_t bucket = -1; int32_t manifest_merge_min_count = 30; + int32_t scan_manifest_entry_cache_max_snapshots = 0; int32_t read_batch_size = 1024; int32_t write_batch_size = 1024; int32_t local_sort_max_num_file_handles = 128; @@ -428,6 +450,9 @@ struct CoreOptions::Impl { bool ignore_delete = false; bool write_buffer_spillable = true; bool write_only = false; + bool bucket_append_ordered = false; + CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = + CoreOptions::SequenceNumberInitMode::SCAN; bool deletion_vectors_enabled = false; bool deletion_vectors_bitmap64 = false; bool force_lookup = false; @@ -442,10 +467,25 @@ struct CoreOptions::Impl { bool row_tracking_enabled = false; bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; + bool variant_infer_shredding_schema = false; + VariantShreddingInferenceMode variant_shredding_inference_mode = + VariantShreddingInferenceMode::PER_FILE; + int32_t variant_shredding_max_schema_width = 300; + int32_t variant_shredding_max_schema_depth = 50; + double variant_shredding_min_field_cardinality_ratio = 0.1; + int32_t variant_shredding_max_infer_buffer_row = 4096; + int32_t variant_shredding_adaptive_max_infer_buffer_row = 256; + double variant_shredding_adaptive_retention_ratio = 0.05; + bool blob_view_resolve_enabled = true; + bool blob_as_descriptor = false; + std::optional blob_split_by_file_size; bool legacy_partition_name_enabled = true; bool global_index_enabled = true; std::optional global_index_thread_num; bool commit_force_compact = false; + bool commit_discard_duplicate_files = false; + bool dynamic_partition_overwrite = true; + bool overwrite_upgrade = true; bool compaction_force_rewrite_all_files = false; bool compaction_force_up_level_0 = false; std::optional global_index_external_path; @@ -484,6 +524,13 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::PAGE_SIZE, &page_size)); // Parse target-file-size - target size of a data file PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::TARGET_FILE_SIZE, &target_file_size)); + // Parse target-file-row-num - target rows of a newly written data file + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::TARGET_FILE_ROW_NUM, &target_file_row_num)); + if (target_file_row_num <= 0) { + return Status::Invalid( + fmt::format("{} should be at least 1", Options::TARGET_FILE_ROW_NUM)); + } // Parse blob.target-file-size - target size of a blob file PAIMON_RETURN_NOT_OK( parser.ParseMemorySize(Options::BLOB_TARGET_FILE_SIZE, &blob_target_file_size)); @@ -520,6 +567,9 @@ struct CoreOptions::Impl { specified_file_system, &file_system)); // Parse write-only - if true, compactions and snapshot expiration will be skipped PAIMON_RETURN_NOT_OK(parser.Parse(Options::WRITE_ONLY, &write_only)); + // Parse bucket-append-ordered - append writes in fixed-bucket mode are ordered + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BUCKET_APPEND_ORDERED, &bucket_append_ordered)); // Parse partition.legacy-name - use legacy ToString for partition names, default true PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTITION_GENERATE_LEGACY_NAME, &legacy_partition_name_enabled)); @@ -560,13 +610,17 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseList(Options::BLOB_VIEW_FIELD, Options::FIELDS_SEPARATOR, &blob_view_fields, /*need_trim=*/true)); - // Parse blob-external-storage-field - descriptor BLOB fields written to external storage - PAIMON_RETURN_NOT_OK(parser.ParseList( - Options::BLOB_EXTERNAL_STORAGE_FIELD, Options::FIELDS_SEPARATOR, - &blob_external_storage_fields, /*need_trim=*/true)); - // Parse blob-external-storage-path - external storage path for configured BLOB fields + // Parse blob-view-upstream-warehouse - warehouse path for configured blob view fields PAIMON_RETURN_NOT_OK( - parser.Parse(Options::BLOB_EXTERNAL_STORAGE_PATH, &blob_external_storage_path)); + parser.Parse(Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, &blob_view_upstream_warehouse)); + // Parse blob-view.resolve.enabled - whether to resolve blob view fields at read time + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BLOB_VIEW_RESOLVE_ENABLED, &blob_view_resolve_enabled)); + // Parse blob-as-descriptor - read blob field as descriptor rather than blob bytes + PAIMON_RETURN_NOT_OK(parser.Parse(Options::BLOB_AS_DESCRIPTOR, &blob_as_descriptor)); + // Parse blob.split-by-file-size - whether blob file size counts in scan splitting + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BLOB_SPLIT_BY_FILE_SIZE, &blob_split_by_file_size)); return Status::OK(); } @@ -642,6 +696,22 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::COMMIT_TIMEOUT, &commit_timeout)); // Parse commit.max-retries - maximum retries when commit failed, default 10 PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_MAX_RETRIES, &commit_max_retries)); + // Parse commit.min-retry-wait - minimum retry wait when commit failed, default 10ms + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MIN_RETRY_WAIT, &commit_min_retry_wait)); + // Parse commit.max-retry-wait - maximum retry wait when commit failed, default 10s + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MAX_RETRY_WAIT, &commit_max_retry_wait)); + // Parse commit.discard-duplicate-files - whether to discard duplicate files on append + PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_DISCARD_DUPLICATE_FILES, + &commit_discard_duplicate_files)); + // Parse dynamic-partition-overwrite - whether overwrite only dynamic partitions + // for partitioned table overwrite. + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::DYNAMIC_PARTITION_OVERWRITE, &dynamic_partition_overwrite)); + // Parse overwrite-upgrade - whether to try upgrading data files after overwrite on + // primary key table + PAIMON_RETURN_NOT_OK(parser.Parse(Options::OVERWRITE_UPGRADE, &overwrite_upgrade)); return Status::OK(); } @@ -652,12 +722,38 @@ struct CoreOptions::Impl { Options::SEQUENCE_FIELD, Options::FIELDS_SEPARATOR, &sequence_field)); // Parse sequence.field.sort-order - order of sequence field, default "ascending" PAIMON_RETURN_NOT_OK(parser.ParseSortOrder(&sequence_field_sort_order)); + // Parse write.sequence-number-init-mode - sequence init mode for write path + std::string write_sequence_init_mode_str = "scan"; + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, &write_sequence_init_mode_str)); + write_sequence_init_mode_str = StringUtils::ToLowerCase(write_sequence_init_mode_str); + if (write_sequence_init_mode_str == "scan") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SCAN; + } else if (write_sequence_init_mode_str == "snapshot") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SNAPSHOT; + } else { + return Status::Invalid(fmt::format("invalid write sequence number init mode: {}", + write_sequence_init_mode_str)); + } // Parse sort-engine - sort engine for primary key table, default "loser-tree" PAIMON_RETURN_NOT_OK(parser.ParseSortEngine(&sort_engine)); // Parse merge-engine - merge engine for primary key table, default "deduplicate" PAIMON_RETURN_NOT_OK(parser.ParseMergeEngine(&merge_engine)); - // Parse ignore-delete - whether to ignore delete records, default false - PAIMON_RETURN_NOT_OK(parser.Parse(Options::IGNORE_DELETE, &ignore_delete)); + // Parse ignore-delete - whether to ignore delete records, default false. + // Java CoreOptions declares first-row.ignore-delete, deduplicate.ignore-delete + // and partial-update.ignore-delete as fallback keys, checked in that order only + // when ignore-delete itself is absent. + std::optional ignore_delete_value; + PAIMON_RETURN_NOT_OK(parser.Parse(Options::IGNORE_DELETE, &ignore_delete_value)); + for (const char* fallback_key : {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, + Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, + Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE}) { + if (ignore_delete_value.has_value()) { + break; + } + PAIMON_RETURN_NOT_OK(parser.Parse(fallback_key, &ignore_delete_value)); + } + ignore_delete = ignore_delete_value.value_or(false); // Parse fields.default-aggregate-function - default agg function for partial-update PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func)); // Parse changelog-producer - whether to double write to a changelog file, default "none" @@ -671,7 +767,6 @@ struct CoreOptions::Impl { // Parse table-read.sequence-number.enabled - expose sequence number in system tables PAIMON_RETURN_NOT_OK(parser.Parse(Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, &table_read_sequence_number_enabled)); - // Parse key-value.sequence_number.enabled - internal sequence number read switch PAIMON_RETURN_NOT_OK(parser.Parse(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, &key_value_sequence_number_enabled)); // Parse partial-update.remove-record-on-sequence-group @@ -714,6 +809,13 @@ struct CoreOptions::Impl { } // Parse scan.mode - scanning behavior of the source, default "default" PAIMON_RETURN_NOT_OK(parser.ParseStartupMode(&startup_mode)); + // Parse scan.manifest-entry-cache.max-snapshots - cached snapshots per bucket. + PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, + &scan_manifest_entry_cache_max_snapshots)); + if (scan_manifest_entry_cache_max_snapshots < 0) { + return Status::Invalid(fmt::format("{} must be non-negative", + Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS)); + } // Parse scan.fallback-branch - fallback branch when partition not found PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_FALLBACK_BRANCH, &scan_fallback_branch)); // Parse branch - branch name, default "main" @@ -793,6 +895,76 @@ struct CoreOptions::Impl { } // Parse lookup configurations: compact mode, bloom filter, remote file, cache, compression. + Status ParseVariantOptions(const ConfigParser& parser) { + // Parse variant.inferShreddingSchema - infer the shredding schema from sampled rows + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_INFER_SHREDDING_SCHEMA, + &variant_infer_shredding_schema)); + PAIMON_RETURN_NOT_OK( + parser.ParseVariantShreddingInferenceMode(&variant_shredding_inference_mode)); + // Parse variant.shredding.maxSchemaWidth - max number of shredded fields, default 300 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH, + &variant_shredding_max_schema_width)); + // Parse variant.shredding.maxSchemaDepth - max shredded nesting depth, default 50 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH, + &variant_shredding_max_schema_depth)); + // Parse variant.shredding.minFieldCardinalityRatio - min occurrence ratio for a field to + // be shredded, default 0.1 + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + &variant_shredding_min_field_cardinality_ratio)); + // Parse variant.shredding.maxInferBufferRow - rows buffered per file for inference, + // default 4096 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, + &variant_shredding_max_infer_buffer_row)); + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW, + &variant_shredding_adaptive_max_infer_buffer_row)); + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO, + &variant_shredding_adaptive_retention_ratio)); + if (variant_shredding_max_schema_width <= 0) { + return Status::Invalid(fmt::format( + "The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH, variant_shredding_max_schema_width)); + } + if (variant_shredding_max_schema_depth <= 0) { + return Status::Invalid(fmt::format( + "The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH, variant_shredding_max_schema_depth)); + } + if (variant_shredding_min_field_cardinality_ratio < 0.0 || + variant_shredding_min_field_cardinality_ratio > 1.0) { + return Status::Invalid( + fmt::format("The option '{}' should be in the range [0, 1], while input is {}", + Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + variant_shredding_min_field_cardinality_ratio)); + } + if (variant_shredding_max_infer_buffer_row <= 0) { + return Status::Invalid( + fmt::format("The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, + variant_shredding_max_infer_buffer_row)); + } + if (variant_shredding_inference_mode == VariantShreddingInferenceMode::ADAPTIVE) { + if (variant_shredding_adaptive_max_infer_buffer_row <= 0) { + return Status::Invalid( + fmt::format("The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW, + variant_shredding_adaptive_max_infer_buffer_row)); + } + if (variant_shredding_adaptive_retention_ratio < 0.0 || + variant_shredding_adaptive_retention_ratio > + variant_shredding_min_field_cardinality_ratio) { + return Status::Invalid( + fmt::format("The option '{}' should be in the range [0, {}], while input is {}", + Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO, + Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + variant_shredding_adaptive_retention_ratio)); + } + } + return Status::OK(); + } + Status ParseLookupOptions(const ConfigParser& parser) { // Parse force-lookup - whether to force lookup for compaction, default false PAIMON_RETURN_NOT_OK(parser.Parse(Options::FORCE_LOOKUP, &force_lookup)); @@ -867,7 +1039,7 @@ Result CoreOptions::FromMap( PAIMON_RETURN_NOT_OK(impl->ParseIndexOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseCompactionOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseLookupOptions(parser)); - + PAIMON_RETURN_NOT_OK(impl->ParseVariantOptions(parser)); return options; } @@ -932,6 +1104,10 @@ int64_t CoreOptions::GetTargetFileSize(bool has_primary_key) const { return impl_->target_file_size.value(); } +int64_t CoreOptions::GetTargetFileRowNum() const { + return impl_->target_file_row_num; +} + int64_t CoreOptions::GetBlobTargetFileSize() const { if (impl_->blob_target_file_size == std::nullopt) { return GetTargetFileSize(/*has_primary_key=*/false); @@ -939,6 +1115,10 @@ int64_t CoreOptions::GetBlobTargetFileSize() const { return impl_->blob_target_file_size.value(); } +bool CoreOptions::BlobSplitByFileSize() const { + return impl_->blob_split_by_file_size.value_or(!impl_->blob_as_descriptor); +} + int64_t CoreOptions::GetCompactionFileSize(bool has_primary_key) const { // file size to join the compaction, we don't process on middle file size to avoid // compact a same file twice (the compression is not calculate so accurately. the output @@ -966,10 +1146,24 @@ std::optional CoreOptions::GetScanSnapshotId() const { std::optional CoreOptions::GetScanTimestampMillis() const { return impl_->scan_timestamp_millis; } + +int32_t CoreOptions::GetScanManifestEntryCacheMaxSnapshots() const { + return impl_->scan_manifest_entry_cache_max_snapshots; +} + int64_t CoreOptions::GetManifestTargetFileSize() const { return impl_->manifest_target_file_size; } +std::shared_ptr CoreOptions::GetCache() const { + return impl_->cache; +} + +CoreOptions& CoreOptions::WithCache(const std::shared_ptr& cache) { + impl_->cache = cache; + return *this; +} + int32_t CoreOptions::GetManifestMergeMinCount() const { return impl_->manifest_merge_min_count; } @@ -1035,6 +1229,26 @@ int32_t CoreOptions::GetCommitMaxRetries() const { return impl_->commit_max_retries; } +int64_t CoreOptions::GetCommitMinRetryWait() const { + return impl_->commit_min_retry_wait; +} + +int64_t CoreOptions::GetCommitMaxRetryWait() const { + return impl_->commit_max_retry_wait; +} + +bool CoreOptions::CommitDiscardDuplicateFiles() const { + return impl_->commit_discard_duplicate_files; +} + +bool CoreOptions::DynamicPartitionOverwrite() const { + return impl_->dynamic_partition_overwrite; +} + +bool CoreOptions::OverwriteUpgrade() const { + return impl_->overwrite_upgrade; +} + int32_t CoreOptions::GetCompactionMinFileNum() const { return impl_->compaction_min_file_num; } @@ -1131,6 +1345,14 @@ bool CoreOptions::WriteOnly() const { return impl_->write_only; } +bool CoreOptions::BucketAppendOrdered() const { + return impl_->bucket_append_ordered; +} + +CoreOptions::SequenceNumberInitMode CoreOptions::WriteSequenceNumberInitMode() const { + return impl_->write_sequence_number_init_mode; +} + std::optional CoreOptions::GetFieldsDefaultFunc() const { return impl_->field_default_func; } @@ -1176,6 +1398,93 @@ Result CoreOptions::FieldCollectAggDistinct(const std::string& field_name) return distinct; } +std::optional CoreOptions::GetVariantShreddingSchema() const { + auto it = impl_->raw_options.find(Options::VARIANT_SHREDDING_SCHEMA); + if (it == impl_->raw_options.end()) { + it = impl_->raw_options.find(Options::PARQUET_VARIANT_SHREDDING_SCHEMA); + } + if (it == impl_->raw_options.end() || it->second.empty()) { + return std::nullopt; + } + return it->second; +} + +bool CoreOptions::VariantInferShreddingSchemaEnabled() const { + return impl_->variant_infer_shredding_schema; +} + +VariantShreddingInferenceMode CoreOptions::GetVariantShreddingInferenceMode() const { + return impl_->variant_shredding_inference_mode; +} + +int32_t CoreOptions::GetVariantShreddingMaxSchemaWidth() const { + return impl_->variant_shredding_max_schema_width; +} + +int32_t CoreOptions::GetVariantShreddingMaxSchemaDepth() const { + return impl_->variant_shredding_max_schema_depth; +} + +double CoreOptions::GetVariantShreddingMinFieldCardinalityRatio() const { + return impl_->variant_shredding_min_field_cardinality_ratio; +} + +int32_t CoreOptions::GetVariantShreddingMaxInferBufferRow() const { + return impl_->variant_shredding_max_infer_buffer_row; +} + +int32_t CoreOptions::GetVariantShreddingAdaptiveMaxInferBufferRow() const { + return impl_->variant_shredding_adaptive_max_infer_buffer_row; +} + +double CoreOptions::GetVariantShreddingAdaptiveRetentionRatio() const { + return impl_->variant_shredding_adaptive_retention_ratio; +} + +Result CoreOptions::GetMapStorageLayout(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_STORAGE_LAYOUT); + PAIMON_ASSIGN_OR_RAISE(std::string layout_str, OptionsUtils::GetValueFromMap( + impl_->raw_options, key, "default")); + std::string lower = StringUtils::ToLowerCase(layout_str); + if (lower == "shared-shredding") { + return MapStorageLayout::SHARED_SHREDDING; + } else if (lower == "default") { + return MapStorageLayout::DEFAULT; + } + return Status::Invalid(fmt::format("invalid map.storage-layout: {}", layout_str)); +} + +Result CoreOptions::GetMapSharedShreddingMaxColumns(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_SHARED_SHREDDING_MAX_COLUMNS); + PAIMON_ASSIGN_OR_RAISE(int32_t max_columns, + OptionsUtils::GetValueFromMap(impl_->raw_options, key, 256)); + if (max_columns <= 0) { + return Status::Invalid(fmt::format("options {} must > 0", + std::string(Options::MAP_SHARED_SHREDDING_MAX_COLUMNS))); + } + return max_columns; +} + +Result +CoreOptions::GetMapSharedShreddingColumnPlacementPolicy(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY); + PAIMON_ASSIGN_OR_RAISE(std::string policy_str, OptionsUtils::GetValueFromMap( + impl_->raw_options, key, "lru")); + std::string lower = StringUtils::ToLowerCase(policy_str); + if (lower == "plain") { + return MapSharedShreddingColumnPlacementPolicy::PLAIN; + } else if (lower == "sequential") { + return MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL; + } else if (lower == "lru") { + return MapSharedShreddingColumnPlacementPolicy::LRU; + } + return Status::Invalid( + fmt::format("invalid map.shared-shredding.column-placement-policy: {}", policy_str)); +} + bool CoreOptions::DeletionVectorsEnabled() const { return impl_->deletion_vectors_enabled; } @@ -1426,6 +1735,14 @@ const std::vector& CoreOptions::GetBlobViewFields() const { return impl_->blob_view_fields; } +std::optional CoreOptions::GetBlobViewUpstreamWarehouse() const { + return impl_->blob_view_upstream_warehouse; +} + +bool CoreOptions::BlobViewResolveEnabled() const { + return impl_->blob_view_resolve_enabled; +} + std::vector CoreOptions::GetBlobInlineFields() const { std::vector blob_inline_fields = impl_->blob_descriptor_fields; blob_inline_fields.insert(blob_inline_fields.end(), impl_->blob_view_fields.begin(), @@ -1433,14 +1750,6 @@ std::vector CoreOptions::GetBlobInlineFields() const { return blob_inline_fields; } -const std::vector& CoreOptions::GetBlobExternalStorageFields() const { - return impl_->blob_external_storage_fields; -} - -std::optional CoreOptions::GetBlobExternalStoragePath() const { - return impl_->blob_external_storage_path; -} - int64_t CoreOptions::GetLookupCacheFileRetentionMs() const { return impl_->lookup_cache_file_retention_ms; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index c047700c..e62a0323 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -26,11 +26,15 @@ #include #include "paimon/bucket/bucket_function_type.h" +#include "paimon/cache/cache.h" +#include "paimon/common/data/variant/variant_defs.h" #include "paimon/core/options/changelog_producer.h" #include "paimon/core/options/compress_options.h" #include "paimon/core/options/external_path_strategy.h" #include "paimon/core/options/lookup_compact_mode.h" #include "paimon/core/options/lookup_strategy.h" +#include "paimon/core/options/map_shared_shredding_column_placement_policy.h" +#include "paimon/core/options/map_storage_layout.h" #include "paimon/core/options/merge_engine.h" #include "paimon/core/options/sort_engine.h" #include "paimon/format/file_format.h" @@ -43,9 +47,19 @@ namespace paimon { class ExpireConfig; +class Cache; class PAIMON_EXPORT CoreOptions { public: + /// Specifies how to initialize the next sequence number for primary key table writers. + enum class SequenceNumberInitMode { + // initialize by scanning existing file metadata. + SCAN, + // initialize from the maximum sequence number recorded in snapshot properties, + // which can avoid scanning existing file metadata in write-only mode. + SNAPSHOT, + }; + static Result FromMap( const std::map& options_map, const std::shared_ptr& specified_file_system = nullptr, @@ -65,7 +79,9 @@ class PAIMON_EXPORT CoreOptions { int32_t GetFileCompressionZstdLevel() const; int64_t GetPageSize() const; int64_t GetTargetFileSize(bool has_primary_key) const; + int64_t GetTargetFileRowNum() const; int64_t GetBlobTargetFileSize() const; + bool BlobSplitByFileSize() const; int64_t GetCompactionFileSize(bool has_primary_key) const; std::string GetPartitionDefaultName() const; @@ -77,8 +93,11 @@ class PAIMON_EXPORT CoreOptions { int64_t GetSourceSplitOpenFileCost() const; std::optional GetScanSnapshotId() const; std::optional GetScanTimestampMillis() const; + int32_t GetScanManifestEntryCacheMaxSnapshots() const; int64_t GetManifestTargetFileSize() const; + std::shared_ptr GetCache() const; + CoreOptions& WithCache(const std::shared_ptr& cache); StartupMode GetStartupMode() const; int32_t GetReadBatchSize() const; @@ -96,6 +115,11 @@ class PAIMON_EXPORT CoreOptions { bool CompactionForceUpLevel0() const; int64_t GetCommitTimeout() const; int32_t GetCommitMaxRetries() const; + int64_t GetCommitMinRetryWait() const; + int64_t GetCommitMaxRetryWait() const; + bool CommitDiscardDuplicateFiles() const; + bool DynamicPartitionOverwrite() const; + bool OverwriteUpgrade() const; int32_t GetCompactionMinFileNum() const; int32_t GetCompactionMaxSizeAmplificationPercent() const; int32_t GetCompactionSizeRatio() const; @@ -111,12 +135,32 @@ class PAIMON_EXPORT CoreOptions { SortEngine GetSortEngine() const; bool IgnoreDelete() const; bool WriteOnly() const; + bool BucketAppendOrdered() const; + SequenceNumberInitMode WriteSequenceNumberInitMode() const; std::optional GetFieldsDefaultFunc() const; Result> GetFieldAggFunc(const std::string& field_name) const; Result FieldAggIgnoreRetract(const std::string& field_name) const; Result FieldListAggDelimiter(const std::string& field_name) const; Result FieldCollectAggDistinct(const std::string& field_name) const; + + Result GetMapStorageLayout(const std::string& field_name) const; + Result GetMapSharedShreddingMaxColumns(const std::string& field_name) const; + Result GetMapSharedShreddingColumnPlacementPolicy( + const std::string& field_name) const; + + /// The configured variant shredding schema JSON, if any (falls back to + /// "parquet.variant.shreddingSchema"). + std::optional GetVariantShreddingSchema() const; + bool VariantInferShreddingSchemaEnabled() const; + VariantShreddingInferenceMode GetVariantShreddingInferenceMode() const; + int32_t GetVariantShreddingMaxSchemaWidth() const; + int32_t GetVariantShreddingMaxSchemaDepth() const; + double GetVariantShreddingMinFieldCardinalityRatio() const; + int32_t GetVariantShreddingMaxInferBufferRow() const; + int32_t GetVariantShreddingAdaptiveMaxInferBufferRow() const; + double GetVariantShreddingAdaptiveRetentionRatio() const; + bool DeletionVectorsEnabled() const; bool DeletionVectorsBitmap64() const; int64_t DeletionVectorTargetFileSize() const; @@ -185,9 +229,9 @@ class PAIMON_EXPORT CoreOptions { const std::vector& GetBlobFields() const; const std::vector& GetBlobDescriptorFields() const; const std::vector& GetBlobViewFields() const; + std::optional GetBlobViewUpstreamWarehouse() const; + bool BlobViewResolveEnabled() const; std::vector GetBlobInlineFields() const; - const std::vector& GetBlobExternalStorageFields() const; - std::optional GetBlobExternalStoragePath() const; const std::map& ToMap() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 39fb1950..f8230c6f 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -42,7 +42,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(64 * 1024L, core_options.GetPageSize()); ASSERT_EQ(256 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/false)); ASSERT_EQ(128 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/true)); + ASSERT_EQ(std::numeric_limits::max(), core_options.GetTargetFileRowNum()); ASSERT_EQ(256 * 1024 * 1024L, core_options.GetBlobTargetFileSize()); + ASSERT_TRUE(core_options.BlobSplitByFileSize()); ASSERT_EQ(187904815, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_EQ(93952404, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); @@ -57,6 +59,8 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(8 * 1024 * 1024L, core_options.GetManifestTargetFileSize()); ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestFullCompactionThresholdSize()); ASSERT_EQ(30, core_options.GetManifestMergeMinCount()); + ASSERT_EQ(0, core_options.GetScanManifestEntryCacheMaxSnapshots()); + ASSERT_EQ(nullptr, core_options.GetCache()); ASSERT_EQ(128 * 1024 * 1024L, core_options.GetSourceSplitTargetSize()); ASSERT_EQ(4 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost()); ASSERT_EQ(1024, core_options.GetReadBatchSize()); @@ -68,8 +72,13 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ("zstd", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(1, core_options.GetSpillCompressOptions().zstd_level); ASSERT_FALSE(core_options.CommitForceCompact()); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + ASSERT_TRUE(core_options.OverwriteUpgrade()); ASSERT_EQ(std::numeric_limits::max(), core_options.GetCommitTimeout()); ASSERT_EQ(10, core_options.GetCommitMaxRetries()); + ASSERT_EQ(10, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(10 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_FALSE(core_options.CommitDiscardDuplicateFiles()); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(10, expire_config.GetSnapshotRetainMin()); ASSERT_EQ(std::numeric_limits::max(), expire_config.GetSnapshotRetainMax()); @@ -82,6 +91,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine()); ASSERT_FALSE(core_options.IgnoreDelete()); ASSERT_FALSE(core_options.WriteOnly()); + ASSERT_FALSE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(5, core_options.GetCompactionMinFileNum()); ASSERT_FALSE(core_options.CompactionForceRewriteAllFiles()); ASSERT_FALSE(core_options.CompactionForceUpLevel0()); @@ -90,6 +102,10 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.FieldAggIgnoreRetract("f1").value()); ASSERT_EQ(",", core_options.FieldListAggDelimiter("f1").value()); ASSERT_FALSE(core_options.FieldCollectAggDistinct("f1").value()); + ASSERT_EQ(MapStorageLayout::DEFAULT, core_options.GetMapStorageLayout("any_col").value()); + ASSERT_EQ(256, core_options.GetMapSharedShreddingMaxColumns("any_col").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + core_options.GetMapSharedShreddingColumnPlacementPolicy("any_col").value()); ASSERT_FALSE(core_options.DeletionVectorsEnabled()); ASSERT_FALSE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); @@ -119,8 +135,8 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.GetBlobDescriptorFields().empty()); ASSERT_TRUE(core_options.GetBlobViewFields().empty()); ASSERT_TRUE(core_options.GetBlobInlineFields().empty()); - ASSERT_TRUE(core_options.GetBlobExternalStorageFields().empty()); - ASSERT_EQ(std::nullopt, core_options.GetBlobExternalStoragePath()); + ASSERT_EQ(std::nullopt, core_options.GetBlobViewUpstreamWarehouse()); + ASSERT_TRUE(core_options.BlobViewResolveEnabled()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPath()); @@ -163,6 +179,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::BUCKET, "3"}, {Options::PAGE_SIZE, "128 kb"}, {Options::TARGET_FILE_SIZE, "512MB"}, + {Options::TARGET_FILE_ROW_NUM, "123"}, {Options::BLOB_TARGET_FILE_SIZE, "1G"}, {Options::PARTITION_DEFAULT_NAME, "foo"}, {Options::MANIFEST_TARGET_FILE_SIZE, "16MB"}, @@ -180,8 +197,14 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::COMMIT_FORCE_COMPACT, "true"}, {Options::COMMIT_TIMEOUT, "120s"}, {Options::COMMIT_MAX_RETRIES, "20"}, + {Options::COMMIT_MIN_RETRY_WAIT, "5ms"}, + {Options::COMMIT_MAX_RETRY_WAIT, "3s"}, + {Options::COMMIT_DISCARD_DUPLICATE_FILES, "true"}, + {Options::DYNAMIC_PARTITION_OVERWRITE, "false"}, + {Options::OVERWRITE_UPGRADE, "false"}, {Options::SCAN_SNAPSHOT_ID, "5"}, {Options::SCAN_MODE, "from-snapshot-full"}, + {Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "7"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "15"}, {Options::SNAPSHOT_NUM_RETAINED_MAX, "30"}, {Options::SNAPSHOT_EXPIRE_LIMIT, "20"}, @@ -222,15 +245,18 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_FIELD, "blob1,blob2"}, {Options::BLOB_DESCRIPTOR_FIELD, "blob3,blob4"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_VIEW_FIELD, "blob5"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "blob3,blob4"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, "FILE:///tmp/blob_view_upstream_warehouse/"}, + {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, {Options::GLOBAL_INDEX_ENABLED, "false"}, {Options::GLOBAL_INDEX_THREAD_NUM, "4"}, {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, {Options::WRITE_ONLY, "true"}, + {Options::BUCKET_APPEND_ORDERED, "true"}, + {Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"}, {Options::COMPACTION_MIN_FILE_NUM, "10"}, {Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"}, {Options::COMPACTION_FORCE_UP_LEVEL_0, "true"}, @@ -262,7 +288,10 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::LOOKUP_REMOTE_LEVEL_THRESHOLD, "2"}, {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}, {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}, - {Options::BUCKET_FUNCTION_TYPE, "mod"}}; + {Options::BUCKET_FUNCTION_TYPE, "mod"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "128"}, + {"fields.metrics.map.shared-shredding.column-placement-policy", "lru"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); auto fs = core_options.GetFileSystem(); @@ -280,11 +309,13 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(128 * 1024L, core_options.GetPageSize()); ASSERT_EQ(512 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/true)); ASSERT_EQ(512 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/false)); + ASSERT_EQ(123, core_options.GetTargetFileRowNum()); ASSERT_EQ(1024 * 1024 * 1024L, core_options.GetBlobTargetFileSize()); ASSERT_EQ("foo", core_options.GetPartitionDefaultName()); ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestTargetFileSize()); ASSERT_EQ(32 * 1024 * 1024L, core_options.GetManifestFullCompactionThresholdSize()); ASSERT_EQ(2, core_options.GetManifestMergeMinCount()); + ASSERT_EQ(nullptr, core_options.GetCache()); ASSERT_EQ(24 * 1024 * 1024L, core_options.GetSourceSplitTargetSize()); ASSERT_EQ(32 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost()); ASSERT_EQ(2048, core_options.GetReadBatchSize()); @@ -296,9 +327,15 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ("lz4", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(2, core_options.GetSpillCompressOptions().zstd_level); ASSERT_TRUE(core_options.CommitForceCompact()); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + ASSERT_FALSE(core_options.OverwriteUpgrade()); ASSERT_EQ(120 * 1000, core_options.GetCommitTimeout()); ASSERT_EQ(20, core_options.GetCommitMaxRetries()); + ASSERT_EQ(5, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(3 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_TRUE(core_options.CommitDiscardDuplicateFiles()); ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1)); + ASSERT_EQ(7, core_options.GetScanManifestEntryCacheMaxSnapshots()); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(15, expire_config.GetSnapshotRetainMin()); ASSERT_EQ(30, expire_config.GetSnapshotRetainMax()); @@ -355,13 +392,13 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.DataEvolutionEnabled()); ASSERT_EQ(core_options.GetBlobFields(), std::vector({"blob1", "blob2"})); ASSERT_EQ(core_options.GetBlobDescriptorFields(), std::vector({"blob3", "blob4"})); + ASSERT_FALSE(core_options.BlobSplitByFileSize()); ASSERT_EQ(core_options.GetBlobViewFields(), std::vector({"blob5"})); ASSERT_EQ(core_options.GetBlobInlineFields(), std::vector({"blob3", "blob4", "blob5"})); - ASSERT_EQ(core_options.GetBlobExternalStorageFields(), - std::vector({"blob3", "blob4"})); - ASSERT_EQ(core_options.GetBlobExternalStoragePath(), - std::optional("FILE:///tmp/blob_external_storage/")); + ASSERT_EQ(core_options.GetBlobViewUpstreamWarehouse(), + std::optional("FILE:///tmp/blob_view_upstream_warehouse/")); + ASSERT_FALSE(core_options.BlobViewResolveEnabled()); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); ASSERT_FALSE(core_options.GlobalIndexEnabled()); ASSERT_EQ(core_options.GetGlobalIndexThreadNum(), 4); @@ -372,6 +409,9 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_TRUE(core_options.WriteOnly()); + ASSERT_TRUE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(10, core_options.GetCompactionMinFileNum()); ASSERT_EQ(123, core_options.GetCompactionMaxSizeAmplificationPercent()); ASSERT_EQ(9, core_options.GetCompactionSizeRatio()); @@ -402,9 +442,16 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.LookupRemoteFileEnabled()); ASSERT_EQ(core_options.GetLookupRemoteLevelThreshold(), 2); ASSERT_EQ(BucketFunctionType::MOD, core_options.GetBucketFunctionType()); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + core_options.GetMapStorageLayout("metrics").value()); + ASSERT_EQ(128, core_options.GetMapSharedShreddingMaxColumns("metrics").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + core_options.GetMapSharedShreddingColumnPlacementPolicy("metrics").value()); } TEST(CoreOptionsTest, TestInvalidCase) { + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::TARGET_FILE_ROW_NUM, "0"}}), + "target-file-row-num should be at least 1"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET, "3.5"}}), "Invalid Config [bucket: 3.5]"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::SCAN_SNAPSHOT_ID, "3.5"}}), @@ -421,11 +468,17 @@ TEST(CoreOptionsTest, TestInvalidCase) { "invalid lookup mode: invalid"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::LOOKUP_COMPACT_MAX_INTERVAL, "invalid"}}), "Invalid Config [lookup-compact.max-interval: invalid]"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "-1"}}), + "scan.manifest-entry-cache.max-snapshots must be non-negative"); ASSERT_NOK_WITH_MSG( CoreOptions::FromMap({{Options::LOOKUP_CACHE_HIGH_PRIO_POOL_RATIO, "1.1"}}), "The high priority pool ratio should in the range [0, 1), while input is 1.1"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET_FUNCTION_TYPE, "invalid"}}), "invalid bucket function type: invalid"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "invalid"}}), + "invalid write sequence number init mode: invalid"); } TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { @@ -437,6 +490,27 @@ TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { ASSERT_EQ(13, core_options.GetLookupCompactMaxInterval()); } +TEST(CoreOptionsTest, TestDynamicPartitionOverwriteOption) { + { + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "false"}})); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "true"}})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } +} + TEST(CoreOptionsTest, TestNumSortedRunsStopTriggerFloorAndDefault) { { std::map options = { @@ -781,6 +855,7 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) { {Options::BUCKET, "3"}, {Options::PAGE_SIZE, "128 kb"}, {Options::TARGET_FILE_SIZE, "512MB"}, + {Options::TARGET_FILE_ROW_NUM, "4321"}, {Options::FILE_FORMAT, "ORC"}, {Options::FILE_COMPRESSION, "lz4"}, {Options::FILE_COMPRESSION_ZSTD_LEVEL, "5"}, @@ -809,6 +884,9 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) { {Options::DATA_FILE_PREFIX, "test-data-"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"}, + {Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW, "77"}, + {Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO, "0.02"}, {Options::BUCKET_FUNCTION_TYPE, "mod"}, }; ASSERT_OK_AND_ASSIGN(CoreOptions source, CoreOptions::FromMap(options)); @@ -822,6 +900,7 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) { // Verify all fields are correctly copied ASSERT_EQ(3, target.GetBucket()); ASSERT_EQ(128 * 1024L, target.GetPageSize()); + ASSERT_EQ(4321, target.GetTargetFileRowNum()); ASSERT_EQ("orc", target.GetFileFormat()->Identifier()); ASSERT_EQ("lz4", target.GetFileCompression()); ASSERT_EQ(5, target.GetFileCompressionZstdLevel()); @@ -850,13 +929,18 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) { ASSERT_EQ("test-data-", target.DataFilePrefix()); ASSERT_TRUE(target.RowTrackingEnabled()); ASSERT_TRUE(target.DataEvolutionEnabled()); + ASSERT_EQ(VariantShreddingInferenceMode::ADAPTIVE, target.GetVariantShreddingInferenceMode()); + ASSERT_EQ(77, target.GetVariantShreddingAdaptiveMaxInferBufferRow()); + ASSERT_DOUBLE_EQ(0.02, target.GetVariantShreddingAdaptiveRetentionRatio()); ASSERT_EQ(BucketFunctionType::MOD, target.GetBucketFunctionType()); // Verify the target's ToMap matches the source's ToMap ASSERT_EQ(source.ToMap(), target.ToMap()); + ASSERT_EQ(source.GetCache(), target.GetCache()); CoreOptions target2 = source; ASSERT_EQ(source.ToMap(), target2.ToMap()); + ASSERT_EQ(source.GetCache(), target2.GetCache()); } TEST(CoreOptionsTest, TestAssignmentIndependence) { @@ -904,5 +988,216 @@ TEST(CoreOptionsTest, TestFallback) { ASSERT_EQ(options.GetBlobDescriptorFields(), std::vector({"new_b1", "new_b2"})); } + { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_SPLIT_BY_FILE_SIZE, "true"}})); + ASSERT_TRUE(options.BlobSplitByFileSize()); + } +} + +TEST(CoreOptionsTest, TestIgnoreDeleteFallbackKeys) { + { + // Tables written by Java may carry first-row.ignore-delete instead of ignore-delete. + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"}, + {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}})); + ASSERT_TRUE(options.IgnoreDelete()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}})); + ASSERT_TRUE(options.IgnoreDelete()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE, "true"}})); + ASSERT_TRUE(options.IgnoreDelete()); + } + { + // The primary key takes precedence over fallback keys, matching Java CoreOptions. + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::IGNORE_DELETE, "false"}, + {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}})); + ASSERT_FALSE(options.IgnoreDelete()); + } + { + // Fallback keys are checked in declaration order. + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "false"}, + {Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}})); + ASSERT_FALSE(options.IgnoreDelete()); + } + { + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "invalid"}}), + "Invalid Config [first-row.ignore-delete: invalid]"); + } +} + +TEST(CoreOptionsTest, TestMapStorageLayout) { + // Test shared-shredding layout configured for a specific column + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.ext_map.map.storage-layout", "shared-shredding"}, + {"fields.ext_map.map.shared-shredding.max-columns", "64"}, + {"fields.normal_map.map.storage-layout", "default"}})); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + options.GetMapStorageLayout("ext_map").value()); + ASSERT_EQ(64, options.GetMapSharedShreddingMaxColumns("ext_map").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("ext_map").value()); + ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("normal_map").value()); + // Unconfigured column falls back to default + ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("other").value()); + ASSERT_EQ(256, options.GetMapSharedShreddingMaxColumns("other").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("other").value()); + } + // Test case-insensitive layout value + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.metrics.map.storage-layout", "Shared-Shredding"}})); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + options.GetMapStorageLayout("metrics").value()); + } + // Test invalid layout value + { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.storage-layout", "invalid"}})); + ASSERT_NOK_WITH_MSG(options.GetMapStorageLayout("col"), + "invalid map.storage-layout: invalid"); + } + // Test invalid max-columns value + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.shared-shredding.max-columns", "0"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingMaxColumns("col"), + "options map.shared-shredding.max-columns must > 0"); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.shared-shredding.max-columns", "-1"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingMaxColumns("col"), + "options map.shared-shredding.max-columns must > 0"); + } + // Test placement policy values + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "PLAIN"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::PLAIN, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "sequential"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "LRU"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "invalid"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingColumnPlacementPolicy("col"), + "invalid map.shared-shredding.column-placement-policy: invalid"); + } } + +TEST(CoreOptionsTest, TestVariantOptions) { + { + // Defaults. + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_EQ(options.GetVariantShreddingSchema(), std::nullopt); + ASSERT_FALSE(options.VariantInferShreddingSchemaEnabled()); + ASSERT_EQ(options.GetVariantShreddingInferenceMode(), + VariantShreddingInferenceMode::PER_FILE); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 300); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 50); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.1); + ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 4096); + ASSERT_EQ(options.GetVariantShreddingAdaptiveMaxInferBufferRow(), 256); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingAdaptiveRetentionRatio(), 0.05); + } + { + // Configured values. + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"variant.shreddingSchema", "{\"type\": \"ROW\"}"}, + {"variant.inferShreddingSchema", "true"}, + {"variant.shredding.inferenceMode", "ADAPTIVE"}, + {"variant.shredding.maxSchemaWidth", "20"}, + {"variant.shredding.maxSchemaDepth", "5"}, + {"variant.shredding.minFieldCardinalityRatio", "0.25"}, + {"variant.shredding.maxInferBufferRow", "128"}, + {"variant.shredding.adaptive.maxInferBufferRow", "64"}, + {"variant.shredding.adaptive.retentionRatio", "0.2"}})); + ASSERT_EQ(options.GetVariantShreddingSchema(), "{\"type\": \"ROW\"}"); + ASSERT_TRUE(options.VariantInferShreddingSchemaEnabled()); + ASSERT_EQ(options.GetVariantShreddingInferenceMode(), + VariantShreddingInferenceMode::ADAPTIVE); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 20); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 5); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.25); + ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 128); + ASSERT_EQ(options.GetVariantShreddingAdaptiveMaxInferBufferRow(), 64); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingAdaptiveRetentionRatio(), 0.2); + } + { + // The legacy parquet-prefixed key is a fallback for the shredding schema. + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{"parquet.variant.shreddingSchema", "{}"}})); + ASSERT_EQ(options.GetVariantShreddingSchema(), "{}"); + } + // Invalid values fail when the options are parsed, not when they are used. + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.inferShreddingSchema", "not_a_bool"}}), + "variant.inferShreddingSchema"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.inferenceMode", "invalid"}}), + "invalid variant shredding inference mode: invalid"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "abc"}}), + "variant.shredding.maxSchemaWidth"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "0"}}), + "should be positive"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaDepth", "-1"}}), + "should be positive"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{"variant.shredding.minFieldCardinalityRatio", "1.5"}}), + "should be in the range [0, 1]"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxInferBufferRow", "0"}}), + "should be positive"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.adaptive.maxInferBufferRow", "0"}}), + "variant.shredding.adaptive.maxInferBufferRow"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.adaptive.retentionRatio", "-0.01"}}), + "variant.shredding.adaptive.retentionRatio"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"}, + {"variant.shredding.adaptive.retentionRatio", "0.11"}}), + "should be in the range [0, variant.shredding.minFieldCardinalityRatio]"); +} + } // namespace paimon::test diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h index ed20c574..1f43468c 100644 --- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h +++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h @@ -84,8 +84,8 @@ class ApplyDeletionVectorBatchReader : public FileBatchReader { return Status::Invalid("ApplyDeletionVectorBatchReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -98,9 +98,15 @@ class ApplyDeletionVectorBatchReader : public FileBatchReader { private: Result Filter(int32_t batch_size) const { - PAIMON_ASSIGN_OR_RAISE(uint64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); - return deletion_vector_->IsValid(previous_batch_first_row_number, batch_size); + RoaringBitmap32 is_valid; + for (int32_t i = 0; i < batch_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, reader_->GetPreviousBatchFileRowId(i)); + PAIMON_ASSIGN_OR_RAISE(bool is_deleted, deletion_vector_->IsDeleted(file_row_id)); + if (!is_deleted) { + is_valid.Add(i); + } + } + return is_valid; } private: diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp index 28920224..daa42d37 100644 --- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp +++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp @@ -51,7 +51,7 @@ class ApplyDeletionVectorBatchReaderTest : public ::testing::Test, pool_ = GetDefaultPool(); fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); } void TearDown() override {} diff --git a/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp b/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp index 533d4acd..ed8c3ba2 100644 --- a/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp +++ b/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp @@ -21,6 +21,7 @@ #include "arrow/util/crc32.h" #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/utils/math.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" @@ -29,10 +30,9 @@ namespace paimon { Result BitmapDeletionVector::SerializeTo(const std::shared_ptr& pool, DataOutputStream* out) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data, SerializeToBytes(pool)); - int64_t size = data->size(); - if (size < 0 || size > std::numeric_limits::max()) { - return Status::Invalid("BitmapDeletionVector serialize size out of range: ", size); - } + size_t size = data->size(); + PAIMON_RETURN_NOT_OK( + ValidateValueInRange(size, "BitmapDeletionVector serialize size")); PAIMON_RETURN_NOT_OK(out->WriteValue(static_cast(size))); PAIMON_RETURN_NOT_OK(out->WriteBytes(data)); uint32_t crc32 = 0; diff --git a/src/paimon/core/deletionvectors/deletion_file_writer.cpp b/src/paimon/core/deletionvectors/deletion_file_writer.cpp index 9942708d..e463dea6 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer.cpp @@ -19,6 +19,7 @@ #include "paimon/core/deletionvectors/deletion_file_writer.h" #include "paimon/common/io/data_output_stream.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" namespace paimon { @@ -43,16 +44,15 @@ Status DeletionFileWriter::Write(const std::string& key, } DataOutputStream output_stream(out_); PAIMON_ASSIGN_OR_RAISE(int32_t length, deletion_vector->SerializeTo(pool_, &output_stream)); - dv_metas_.insert(key, DeletionVectorMeta(key, static_cast(start), length, - deletion_vector->GetCardinality())); + dv_metas_.insert_or_assign(key, DeletionVectorMeta(key, static_cast(start), length, + deletion_vector->GetCardinality())); return Status::OK(); } Result> DeletionFileWriter::GetResult() const { - int64_t length = output_bytes_; - if (length < 0 || length > std::numeric_limits::max()) { + if (output_bytes_ < 0 || output_bytes_ > std::numeric_limits::max()) { return Status::Invalid( - fmt::format("Deletion file result length {} out of int32 range.", length)); + fmt::format("Deletion file result length {} out of int32 range.", output_bytes_)); } std::optional final_path; if (is_external_path_) { @@ -60,8 +60,8 @@ Result> DeletionFileWriter::GetResult() const { final_path = external_path.ToString(); } return std::make_unique(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, - PathUtil::GetName(path_), length, dv_metas_.size(), - dv_metas_, final_path); + PathUtil::GetName(path_), output_bytes_, + dv_metas_.size(), dv_metas_, final_path); } } // namespace paimon diff --git a/src/paimon/core/deletionvectors/deletion_file_writer.h b/src/paimon/core/deletionvectors/deletion_file_writer.h index 9a04b235..5c48490c 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer.h +++ b/src/paimon/core/deletionvectors/deletion_file_writer.h @@ -21,9 +21,7 @@ #include #include -#include "fmt/format.h" #include "paimon/common/utils/linked_hash_map.h" -#include "paimon/common/utils/path_util.h" #include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/index/deletion_vector_meta.h" #include "paimon/core/index/index_path_factory.h" diff --git a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp index 93777f03..140f17f1 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp @@ -94,7 +94,40 @@ TEST(DeletionFileWriterTest, GetResultWithoutCloseShouldFail) { auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto writer, DeletionFileWriter::Create(path_factory, fs, pool)); - ASSERT_NOK_WITH_MSG(writer->GetResult(), "result length -1 out of int32 range"); + ASSERT_NOK_WITH_MSG(writer->GetResult(), "Deletion file result length -1 out of int32 range"); +} + +TEST(DeletionFileWriterTest, WriteOverwritesDuplicateDataFileName) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + FileSystemFactory::Get("local", dir->Str(), {})); + auto path_factory = std::make_shared(dir->Str()); + auto pool = GetDefaultPool(); + + ASSERT_OK_AND_ASSIGN(auto writer, DeletionFileWriter::Create(path_factory, fs, pool)); + + RoaringBitmap32 roaring_1; + roaring_1.Add(1); + auto dv_1 = std::make_shared(roaring_1); + + RoaringBitmap32 roaring_2; + roaring_2.Add(2); + roaring_2.Add(3); + auto dv_2 = std::make_shared(roaring_2); + + ASSERT_OK(writer->Write("data-file-1", dv_1)); + ASSERT_OK(writer->Write("data-file-1", dv_2)); + ASSERT_OK(writer->Close()); + + ASSERT_OK_AND_ASSIGN(auto meta, writer->GetResult()); + const auto& dv_ranges = meta->DvRanges(); + ASSERT_TRUE(dv_ranges.has_value()); + ASSERT_EQ(dv_ranges->size(), 1); + + auto iter = dv_ranges->find("data-file-1"); + ASSERT_NE(iter, dv_ranges->end()); + ASSERT_GT(iter->second.GetOffset(), 1); + ASSERT_EQ(iter->second.GetCardinality(), std::optional(2)); } TEST(DeletionFileWriterTest, ExternalPathInResult) { diff --git a/src/paimon/core/deletionvectors/deletion_vector.cpp b/src/paimon/core/deletionvectors/deletion_vector.cpp index 82e347d7..f981d19b 100644 --- a/src/paimon/core/deletionvectors/deletion_vector.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector.cpp @@ -18,6 +18,7 @@ */ #include "paimon/core/deletionvectors/deletion_vector.h" +#include #include #include @@ -61,6 +62,22 @@ DeletionVector::Factory DeletionVector::CreateFactory( }; } +std::unordered_map DeletionVector::CreateDeletionFileMap( + const std::vector>& data_files, + const std::vector>& deletion_files) { + std::unordered_map deletion_file_map; + if (deletion_files.empty()) { + return deletion_file_map; + } + assert(deletion_files.size() == data_files.size()); + for (size_t i = 0; i < deletion_files.size(); i++) { + if (deletion_files[i] != std::nullopt) { + deletion_file_map.insert_or_assign(data_files[i]->file_name, deletion_files[i].value()); + } + } + return deletion_file_map; +} + Result> DeletionVector::DeserializeFromBytes(const Bytes* bytes, MemoryPool* pool) { return BitmapDeletionVector::Deserialize(bytes->data(), bytes->size(), pool); diff --git a/src/paimon/core/deletionvectors/deletion_vector.h b/src/paimon/core/deletionvectors/deletion_vector.h index bc53af2e..e2757c63 100644 --- a/src/paimon/core/deletionvectors/deletion_vector.h +++ b/src/paimon/core/deletionvectors/deletion_vector.h @@ -51,6 +51,14 @@ class DeletionVector { static Factory CreateFactory(const std::shared_ptr& dv_maintainer); + /// Builds a map from data file name to its deletion file. + /// + /// Entries whose deletion file is absent are skipped. Returns an empty map when + /// `deletion_files` is empty. + static std::unordered_map CreateDeletionFileMap( + const std::vector>& data_files, + const std::vector>& deletion_files); + virtual ~DeletionVector() = default; /// Marks the row at the specified position as deleted. diff --git a/src/paimon/core/deletionvectors/deletion_vector_test.cpp b/src/paimon/core/deletionvectors/deletion_vector_test.cpp index e8742cd3..daf2ecf1 100644 --- a/src/paimon/core/deletionvectors/deletion_vector_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector_test.cpp @@ -21,12 +21,16 @@ #include #include #include +#include +#include #include #include #include "gtest/gtest.h" #include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/table/source/deletion_file.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/byte_order.h" #include "paimon/io/data_input_stream.h" @@ -43,6 +47,16 @@ void AppendInt32BigEndian(std::vector* bytes, int32_t value) { bytes->push_back(static_cast(value & 0xFF)); } +std::shared_ptr CreateDataFileMeta(const std::string& file_name) { + return std::make_shared( + file_name, /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, + DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0), + std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, + std::nullopt); +} + } // namespace TEST(DeletionVectorTest, TestSimple) { @@ -163,4 +177,36 @@ TEST(DeletionVectorTest, ReadFromDataInputStreamInvalidMagicNumber) { "Invalid magic number"); } +TEST(DeletionVectorTest, CreateDeletionFileMap) { + std::vector> data_files = {CreateDataFileMeta("file-0.orc"), + CreateDataFileMeta("file-1.orc"), + CreateDataFileMeta("file-2.orc")}; + + auto empty_map = DeletionVector::CreateDeletionFileMap(data_files, {}); + ASSERT_TRUE(empty_map.empty()); + + DeletionFile deletion_file_0("dv-0", /*offset=*/10, /*length=*/20, /*cardinality=*/3); + DeletionFile deletion_file_2("dv-2", /*offset=*/30, /*length=*/40, std::nullopt); + std::vector> deletion_files = {deletion_file_0, std::nullopt, + deletion_file_2}; + + auto deletion_file_map = DeletionVector::CreateDeletionFileMap(data_files, deletion_files); + ASSERT_EQ(deletion_file_map.size(), 2); + ASSERT_EQ(deletion_file_map.at("file-0.orc"), deletion_file_0); + ASSERT_EQ(deletion_file_map.count("file-1.orc"), 0); + ASSERT_EQ(deletion_file_map.at("file-2.orc"), deletion_file_2); + + DeletionFile deletion_file_0_new("dv-0-new", /*offset=*/50, /*length=*/60, + /*cardinality=*/7); + std::vector> duplicate_data_files = { + CreateDataFileMeta("file-0.orc"), CreateDataFileMeta("file-0.orc")}; + std::vector> duplicate_deletion_files = {deletion_file_0, + deletion_file_0_new}; + + auto duplicate_deletion_file_map = + DeletionVector::CreateDeletionFileMap(duplicate_data_files, duplicate_deletion_files); + ASSERT_EQ(duplicate_deletion_file_map.size(), 1); + ASSERT_EQ(duplicate_deletion_file_map.at("file-0.orc"), deletion_file_0_new); +} + } // namespace paimon::test diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 83f12395..e4b3e7b7 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -23,6 +23,7 @@ #include #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "paimon/common/global_index/offset_global_index_reader.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/utils/scope_guard.h" @@ -52,6 +53,21 @@ Result> GlobalIndexScanImpl::Create( const Snapshot& snapshot, const std::shared_ptr& partitions, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) { + auto final_executor = executor; + if (!final_executor) { + std::optional thread_num = options.GetGlobalIndexThreadNum(); + if (thread_num) { + if (thread_num.value() <= 0) { + return Status::Invalid( + fmt::format("invalid global index thread number {}", thread_num.value())); + } + } else { + uint32_t cpu_count = std::thread::hardware_concurrency(); + thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; + } + PAIMON_ASSIGN_OR_RAISE(final_executor, + CreateDefaultExecutor(static_cast(thread_num.value()))); + } auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, @@ -103,15 +119,6 @@ Result> GlobalIndexScanImpl::Create( index_metas[index_meta->index_field_id][index_file_meta->IndexType()][range].push_back( index_file_meta); } - auto final_executor = executor; - if (!final_executor) { - std::optional thread_num = options.GetGlobalIndexThreadNum(); - if (!thread_num) { - uint32_t cpu_count = std::thread::hardware_concurrency(); - thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; - } - final_executor = CreateDefaultExecutor(static_cast(thread_num.value())); - } return std::unique_ptr(new GlobalIndexScanImpl( table_schema, options, path_factory, std::move(index_metas), final_executor, pool)); } @@ -182,7 +189,6 @@ Result>> GlobalIndexScanImpl::Cre if (row_range_index && !row_range_index->Intersects(range.from, range.to)) { continue; } - // TODO(xinyu.lxy): c_arrow_schema may contains additional associated fields. auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index a42f39a1..f0e35050 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -19,11 +19,18 @@ #include "paimon/global_index/global_index_write_task.h" +#include + +#include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/type.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/casting/casting_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_file_manager.h" #include "paimon/core/io/data_increment.h" @@ -38,6 +45,17 @@ #include "paimon/table/source/table_read.h" namespace paimon { namespace { +Result> CreateGlobalIndexer(const std::string& index_type, + const CoreOptions& core_options) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, core_options.ToMap())); + if (!indexer) { + return Status::Invalid( + fmt::format("Unknown index type {}, may not registered", index_type)); + } + return indexer; +} + Result> CreateGlobalIndexFileManager( const std::string& table_path, const std::shared_ptr& table_schema, const CoreOptions& core_options, const std::shared_ptr& pool) { @@ -61,25 +79,81 @@ Result> CreateGlobalIndexFileManager( } Result> CreateGlobalIndexWriter( - const std::string& index_type, const DataField& field, + const GlobalIndexer& indexer, const DataField& field, + const std::vector& extra_fields, const std::shared_ptr& index_file_manager, - const CoreOptions& core_options, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, - GlobalIndexerFactory::Get(index_type, core_options.ToMap())); - if (!indexer) { - return Status::Invalid( - fmt::format("Unknown index type {}, may not registered", index_type)); + const std::shared_ptr& pool) { + arrow::FieldVector arrow_fields; + arrow_fields.reserve(extra_fields.size() + 1); + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(field)); + for (const auto& extra_field : extra_fields) { + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(extra_field)); } - // TODO(xinyu.lxy): may add additional fields to read for index write - auto arrow_field = DataField::ConvertDataFieldToArrowField(field); - auto arrow_schema = arrow::schema({arrow_field}); + auto arrow_schema = arrow::schema(arrow_fields); ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); - return indexer->CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); + ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + return indexer.CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); +} + +Result> GetExtraFields(const TableSchema& table_schema, + const std::string& field_name, + const std::vector& extra_field_names) { + std::vector extra_fields; + extra_fields.reserve(extra_field_names.size()); + std::set dedup_field_names; + for (const auto& extra_field_name : extra_field_names) { + if (extra_field_name == field_name) { + return Status::Invalid(fmt::format( + "global index extra field {} must not be the indexed field", extra_field_name)); + } + if (!dedup_field_names.insert(extra_field_name).second) { + return Status::Invalid(fmt::format("global index extra field {} must not be duplicated", + extra_field_name)); + } + PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema.GetField(extra_field_name)); + extra_fields.push_back(extra_field); + } + return extra_fields; +} + +std::vector BuildReadFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector read_field_names; + read_field_names.reserve(extra_fields.size() + 2); + read_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + read_field_names.push_back(extra_field.Name()); + } + read_field_names.push_back(SpecialFields::RowId().Name()); + return read_field_names; +} + +std::vector BuildWriterFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector writer_field_names; + writer_field_names.reserve(extra_fields.size() + 1); + writer_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + writer_field_names.push_back(extra_field.Name()); + } + return writer_field_names; +} + +std::optional> GetExtraFieldIds(const std::vector& extra_fields) { + if (extra_fields.empty()) { + return std::nullopt; + } + std::vector extra_field_ids; + extra_field_ids.reserve(extra_fields.size()); + for (const auto& extra_field : extra_fields) { + extra_field_ids.push_back(extra_field.Id()); + } + return extra_field_ids; } Result> CreateBatchReader( - const std::string& table_path, const std::string& field_name, + const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { ReadContextBuilder read_context_builder(table_path); @@ -87,7 +161,7 @@ Result> CreateBatchReader( .WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) - .SetReadSchema({field_name, SpecialFields::RowId().Name()}); + .SetReadFieldNames(read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, @@ -95,9 +169,99 @@ Result> CreateBatchReader( return table_read->CreateReader(indexed_split); } -Result> BuildIndex(const std::string& field_name, const Range& range, - BatchReader* batch_reader, - GlobalIndexWriter* global_index_writer) { +Result> CastDictionaryArrayToString( + const std::shared_ptr& array, arrow::MemoryPool* pool) { + arrow::Type::type type_id = array->type_id(); + if (type_id == arrow::Type::DICTIONARY) { + const auto* dictionary_type = + static_cast(array->type().get()); + arrow::Type::type value_type = dictionary_type->value_type()->id(); + if (value_type != arrow::Type::STRING && value_type != arrow::Type::LARGE_STRING) { + return Status::Invalid(fmt::format( + "GlobalIndexWriteTask cannot decode dictionary array with value type {}", + dictionary_type->value_type()->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr casted_array, + CastingUtils::Cast(array, arrow::utf8(), arrow::compute::CastOptions::Safe(), pool)); + return casted_array; + } + if (type_id != arrow::Type::STRUCT && type_id != arrow::Type::MAP && + type_id != arrow::Type::LIST) { + return array; + } + + if (type_id == arrow::Type::STRUCT) { + std::shared_ptr struct_array = + std::static_pointer_cast(array); + arrow::ArrayVector children; + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + std::shared_ptr child = struct_array->field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_child, + CastDictionaryArrayToString(child, pool)); + if (casted_child != child && children.empty()) { + children = struct_array->fields(); + } + if (!children.empty()) { + children[i] = std::move(casted_child); + } + } + if (children.empty()) { + return array; + } + std::vector field_names; + field_names.reserve(struct_array->num_fields()); + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + field_names.push_back(struct_array->struct_type()->field(i)->name()); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr casted_array, + arrow::StructArray::Make(children, field_names, struct_array->null_bitmap(), + struct_array->null_count(), struct_array->offset())); + return casted_array; + } + + if (type_id == arrow::Type::MAP) { + std::shared_ptr map_array = + std::static_pointer_cast(array); + std::shared_ptr original_keys = map_array->keys(); + std::shared_ptr original_items = map_array->items(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr keys, + CastDictionaryArrayToString(original_keys, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr items, + CastDictionaryArrayToString(original_items, pool)); + if (keys == original_keys && items == original_items) { + return array; + } + const auto* map_type = static_cast(map_array->type().get()); + std::shared_ptr casted_type = std::make_shared( + map_type->key_field()->WithType(keys->type()), + map_type->item_field()->WithType(items->type()), map_type->keys_sorted()); + return std::make_shared( + casted_type, map_array->length(), map_array->value_offsets(), keys, items, + map_array->null_bitmap(), map_array->null_count(), map_array->offset()); + } + + std::shared_ptr list_array = + std::static_pointer_cast(array); + std::shared_ptr original_values = list_array->values(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + CastDictionaryArrayToString(original_values, pool)); + if (values == original_values) { + return array; + } + const auto* list_type = static_cast(list_array->type().get()); + std::shared_ptr casted_type = + arrow::list(list_type->value_field()->WithType(values->type())); + return std::make_shared( + casted_type, list_array->length(), list_array->value_offsets(), values, + list_array->null_bitmap(), list_array->null_count(), list_array->offset()); +} + +Result> BuildIndex( + const std::string& field_name, const Range& range, + const std::vector& writer_field_names, BatchReader* batch_reader, + GlobalIndexWriter* global_index_writer, arrow::MemoryPool* arrow_pool) { while (true) { PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch read_batch, batch_reader->NextBatch()); if (BatchReader::IsEofBatch(read_batch)) { @@ -111,11 +275,6 @@ Result> BuildIndex(const std::string& field_name, return Status::Invalid( "array read from batch reader is not a struct array in GlobalIndexWriteTask"); } - auto indexed_array = struct_array->GetFieldByName(field_name); - if (!indexed_array) { - return Status::Invalid(fmt::format( - "read array does not contain {} field in GlobalIndexWriteTask", field_name)); - } auto row_id_array = struct_array->GetFieldByName(SpecialFields::RowId().Name()); auto typed_row_id_array = std::dynamic_pointer_cast(row_id_array); if (!typed_row_id_array) { @@ -134,8 +293,22 @@ Result> BuildIndex(const std::string& field_name, } relative_row_ids.push_back(row_id - range.from); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_array, - arrow::StructArray::Make({indexed_array}, {field_name})); + std::vector> writer_arrays; + writer_arrays.reserve(writer_field_names.size()); + for (const auto& writer_field_name : writer_field_names) { + auto writer_array = struct_array->GetFieldByName(writer_field_name); + if (!writer_array) { + return Status::Invalid( + fmt::format("read array does not contain {} field in GlobalIndexWriteTask", + writer_field_name)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr decoded_writer_array, + CastDictionaryArrayToString(writer_array, arrow_pool)); + writer_arrays.push_back(std::move(decoded_writer_array)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr new_array, + arrow::StructArray::Make(writer_arrays, writer_field_names)); ::ArrowArray c_new_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*new_array, &c_new_array)); PAIMON_RETURN_NOT_OK( @@ -147,7 +320,8 @@ Result> BuildIndex(const std::string& field_name, Result> ToCommitMessage( const std::string& index_type, int32_t field_id, const Range& range, const std::vector& global_index_io_metas, const BinaryRow& partition, - int32_t bucket, const std::shared_ptr& file_manager) { + int32_t bucket, const std::shared_ptr& file_manager, + const std::optional>& extra_field_ids) { std::vector> index_file_metas; index_file_metas.reserve(global_index_io_metas.size()); bool is_external_path = file_manager->IsExternalPath(); @@ -160,8 +334,7 @@ Result> ToCommitMessage( index_file_metas.push_back(std::make_shared( index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(), /*dv_ranges=*/std::nullopt, external_path, - GlobalIndexMeta(range.from, range.to, field_id, - /*extra_field_ids=*/std::nullopt, io_meta.metadata))); + GlobalIndexMeta(range.from, range.to, field_id, extra_field_ids, io_meta.metadata))); } DataIncrement data_increment(std::move(index_file_metas)); return std::make_shared(partition, bucket, @@ -185,6 +358,7 @@ Result> GlobalIndexWriteTask::WriteIndex( } const auto& range = ranges[0]; std::shared_ptr pool = memory_pool ? memory_pool : GetDefaultPool(); + std::unique_ptr arrow_pool = GetArrowPool(pool); // load schema PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(options, file_system)); @@ -202,6 +376,17 @@ Result> GlobalIndexWriteTask::WriteIndex( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(final_options, file_system)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + CreateGlobalIndexer(index_type, core_options)); + PAIMON_ASSIGN_OR_RAISE(std::optional> extra_field_names, + indexer->GetExtraFieldNames()); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); + PAIMON_ASSIGN_OR_RAISE(std::vector extra_fields, + GetExtraFields(*table_schema, field_name, + extra_field_names.value_or(std::vector()))); + std::optional> extra_field_ids = GetExtraFieldIds(extra_fields); + std::vector writer_field_names = BuildWriterFieldNames(field_name, extra_fields); + std::vector read_field_names = BuildReadFieldNames(field_name, extra_fields); // create index file manager PAIMON_ASSIGN_OR_RAISE( @@ -211,13 +396,12 @@ Result> GlobalIndexWriteTask::WriteIndex( // create batch reader PAIMON_ASSIGN_OR_RAISE( std::unique_ptr batch_reader, - CreateBatchReader(table_path, field_name, indexed_split, core_options, pool)); + CreateBatchReader(table_path, read_field_names, indexed_split, core_options, pool)); // create global index writer - PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr global_index_writer, - CreateGlobalIndexWriter(index_type, field, index_file_manager, core_options, pool)); + CreateGlobalIndexWriter(*indexer, field, extra_fields, index_file_manager, pool)); ScopeGuard guard([&]() { global_index_writer.reset(); @@ -225,13 +409,14 @@ Result> GlobalIndexWriteTask::WriteIndex( }); // read from data split and write to index writer - PAIMON_ASSIGN_OR_RAISE( - std::vector global_index_io_metas, - BuildIndex(field_name, range, batch_reader.get(), global_index_writer.get())); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_io_metas, + BuildIndex(field_name, range, writer_field_names, batch_reader.get(), + global_index_writer.get(), arrow_pool.get())); // generate commit message return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas, - data_split->Partition(), data_split->Bucket(), index_file_manager); + data_split->Partition(), data_split->Bucket(), index_file_manager, + extra_field_ids); } } // namespace paimon diff --git a/src/paimon/core/index/index_file.h b/src/paimon/core/index/index_file.h index e76c19d9..f3829b4e 100644 --- a/src/paimon/core/index/index_file.h +++ b/src/paimon/core/index/index_file.h @@ -41,11 +41,11 @@ class IndexFile { return path_factory_->ToPath(file); } - virtual Result FileSize(const std::shared_ptr& file) const { + virtual Result FileSize(const std::shared_ptr& file) const { return FileSize(Path(file)); } - virtual Result FileSize(const std::string& file) const { + virtual Result FileSize(const std::string& file) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(file)); return file_status->GetLen(); } diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index 8236d184..2f839b8d 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -268,9 +268,9 @@ TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) { snapshot.DeltaManifestListSize(), snapshot.ChangelogManifestList(), snapshot.ChangelogManifestListSize(), /*index_manifest=*/std::nullopt, snapshot.CommitUser(), snapshot.CommitIdentifier(), snapshot.GetCommitKind(), - snapshot.TimeMillis(), snapshot.LogOffsets(), snapshot.TotalRecordCount(), - snapshot.DeltaRecordCount(), snapshot.ChangelogRecordCount(), snapshot.Watermark(), - snapshot.Statistics(), snapshot.Properties(), snapshot.NextRowId()); + snapshot.TimeMillis(), snapshot.TotalRecordCount(), snapshot.DeltaRecordCount(), + snapshot.ChangelogRecordCount(), snapshot.Watermark(), snapshot.Statistics(), + snapshot.Properties(), snapshot.NextRowId()); auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get()); std::unordered_set partitions = {partition}; diff --git a/src/paimon/core/io/append_data_file_writer_factory.cpp b/src/paimon/core/io/append_data_file_writer_factory.cpp new file mode 100644 index 00000000..e12374dd --- /dev/null +++ b/src/paimon/core/io/append_data_file_writer_factory.cpp @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/append_data_file_writer_factory.h" + +#include + +#include "arrow/c/abi.h" +#include "arrow/c/helpers.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +AppendDataFileWriterFactory::AppendDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + write_schema_(write_schema), + write_cols_(write_cols), + seq_num_counter_(seq_num_counter), + file_source_(file_source), + path_factory_(path_factory) {} + +std::shared_ptr AppendDataFileWriterFactory::ResolveSeqNumCounter() const { + return options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; +} + +Result>>> +AppendDataFileWriterFactory::CreateWriter() const { + std::shared_ptr seq_num_counter = ResolveSeqNumCounter(); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*options_.GetFileFormat(), write_schema_, + /*create_stats_extractor=*/true)); + auto writer = std::make_unique( + options_.GetFileCompression(), std::function(), + schema_id_, seq_num_counter, file_source_, resources.stats_extractor, + path_factory_->IsExternalPath(), write_cols_, pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/append_data_file_writer_factory.h b/src/paimon/core/io/append_data_file_writer_factory.h new file mode 100644 index 00000000..e5e9c3d6 --- /dev/null +++ b/src/paimon/core/io/append_data_file_writer_factory.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class LongCounter; +class MemoryPool; + +class AppendDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { + public: + AppendDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + protected: + // Resolves the sequence-number counter for a newly created writer. When data evolution is + // enabled each file gets a fresh counter starting at 0; otherwise the factory's shared counter + // is reused. + std::shared_ptr ResolveSeqNumCounter() const; + + std::shared_ptr write_schema_; + std::optional> write_cols_; + std::shared_ptr seq_num_counter_; + FileSource file_source_; + std::shared_ptr path_factory_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp index fc8b54a6..1792b43c 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp @@ -40,7 +40,7 @@ AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( pool_(pool), sort_merge_reader_(std::move(sort_merge_reader)), create_consumer_(std::move(create_consumer)) { - kv_queue_.set_capacity(batch_size); + kv_queue_.set_capacity(consumer_thread_num * 2); result_queue_.set_capacity(RESULT_BATCH_COUNT); } @@ -81,8 +81,8 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { Result>> consumer = create_consumer_(); PAIMON_RETURN_NOT_OK(consumer.status()); auto async_consumer = std::make_unique>( - batch_size_ / consumer_thread_num_, std::move(consumer).value(), consume_finished_, - consumer_finished_count_, kv_queue_, result_queue_); + std::move(consumer).value(), consume_finished_, consumer_finished_count_, kv_queue_, + result_queue_); consumers_.push_back(std::move(async_consumer)); } } @@ -109,24 +109,33 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { template Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { + std::vector batch; + batch.reserve(batch_size_); while (!consume_finished_) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, sort_merge_reader_->NextBatch()); if (iterator == nullptr) { - // all iterator is all visited - kv_queue_.push(std::nullopt); break; } while (!consume_finished_) { PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); if (!has_next) { - // current iterator is all visited break; } - std::optional kv = std::move(iterator->Next()); - kv_queue_.push(std::move(kv)); + batch.push_back(std::move(iterator->Next())); + if (static_cast(batch.size()) >= batch_size_) { + kv_queue_.push(std::move(batch)); + batch = std::vector(); + batch.reserve(batch_size_); + } } } + // Push remaining rows + if (!batch.empty()) { + kv_queue_.push(std::move(batch)); + } + // Push empty batch as EOF signal + kv_queue_.push(std::vector()); return Status::OK(); } @@ -159,8 +168,8 @@ void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { } } - std::optional kv; - while (kv_queue_.try_pop(kv)) { + std::vector kv_batch; + while (kv_queue_.try_pop(kv_batch)) { } } @@ -169,12 +178,11 @@ template class AsyncKeyValueProducerAndConsumer; template AsyncKeyValueConsumer::AsyncKeyValueConsumer( - int32_t batch_size, std::unique_ptr>&& key_value_consumer, + std::unique_ptr>&& key_value_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, + tbb::concurrent_bounded_queue>& kv_queue, tbb::concurrent_bounded_queue& result_queue) - : batch_size_(batch_size), - key_value_consumer_(std::move(key_value_consumer)), + : key_value_consumer_(std::move(key_value_consumer)), consume_finished_(consume_finished), consumer_finished_count_(consumer_finished_count), kv_queue_(kv_queue), @@ -197,30 +205,18 @@ Status AsyncKeyValueConsumer::GetStatus() const { template Status AsyncKeyValueConsumer::ConsumeLoop() { while (!consume_finished_) { - int32_t cur_batch_size = 0; std::vector key_value_vec; - key_value_vec.reserve(batch_size_); - while (!consume_finished_) { - std::optional kv; - if (!kv_queue_.try_pop(kv)) { - usleep(1); - continue; - } - if (!kv) { - consume_finished_ = true; - break; - } - key_value_vec.push_back(std::move(kv).value()); - cur_batch_size++; - if (cur_batch_size >= batch_size_) { - break; - } + if (!kv_queue_.try_pop(key_value_vec)) { + usleep(100); + continue; } - - if (cur_batch_size > 0) { - PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); - result_queue_.push(std::move(result)); + if (key_value_vec.empty()) { + // Empty batch is EOF signal; re-push for other consumers + kv_queue_.push(std::move(key_value_vec)); + break; } + PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); + result_queue_.push(std::move(result)); } consumer_finished_count_++; return Status::OK(); diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h index 235b2912..af8bbed6 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.h +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h @@ -94,18 +94,17 @@ class AsyncKeyValueProducerAndConsumer { std::shared_future producer_future_; std::vector>> consumers_; std::atomic consumer_finished_count_ = 0; - tbb::concurrent_bounded_queue> kv_queue_; + tbb::concurrent_bounded_queue> kv_queue_; tbb::concurrent_bounded_queue result_queue_; }; template class AsyncKeyValueConsumer { public: - AsyncKeyValueConsumer(int32_t batch_size, - std::unique_ptr>&& key_value_consumer, + AsyncKeyValueConsumer(std::unique_ptr>&& key_value_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, + tbb::concurrent_bounded_queue>& kv_queue, tbb::concurrent_bounded_queue& result_queue); ~AsyncKeyValueConsumer() { @@ -119,12 +118,11 @@ class AsyncKeyValueConsumer { Status ConsumeLoop(); private: - int32_t batch_size_; std::unique_ptr> key_value_consumer_; std::shared_future consumer_future_; std::atomic& consume_finished_; std::atomic& consumer_finished_count_; - tbb::concurrent_bounded_queue>& kv_queue_; + tbb::concurrent_bounded_queue>& kv_queue_; tbb::concurrent_bounded_queue& result_queue_; }; diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp new file mode 100644 index 00000000..b6d9346c --- /dev/null +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/blob_data_file_writer_factory.h" + +#include +#include +#include +#include + +#include "paimon/common/data/blob_defs.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +BlobDataFileWriterFactory::BlobDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& file_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, bool write_placeholder, + const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + file_schema_(file_schema), + write_cols_(write_cols), + seq_num_counter_(seq_num_counter), + path_factory_(path_factory), + write_placeholder_(write_placeholder) {} + +Result>>> +BlobDataFileWriterFactory::CreateWriter() const { + std::shared_ptr seq_num_counter = + options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::map format_options = options_.ToMap(); + // The placeholder channel is internal: strip user-supplied blob.internal.* table options so + // only the writer's own decision below can enable it. + BlobDefs::EraseInternalPlaceholderOptions(&format_options); + if (write_placeholder_) { + format_options[BlobDefs::kWritePlaceholderKey] = "true"; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get("blob", format_options)); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*format, file_schema_, + /*create_stats_extractor=*/true)); + auto writer = std::make_unique( + /*compression=*/"none", std::function(), schema_id_, + seq_num_counter, FileSource::Append(), resources.stats_extractor, + path_factory_->IsExternalPath(), write_cols_, pool_); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), path_factory_->NewBlobPath(), + resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h new file mode 100644 index 00000000..63c84d72 --- /dev/null +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class LongCounter; +class MemoryPool; + +class BlobDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { + public: + /// `write_placeholder` marks a data-evolution partial-update write: the blob format + /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder + /// sentinel rows as placeholder entries. + BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& file_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, + bool write_placeholder, const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + private: + std::shared_ptr file_schema_; + std::optional> write_cols_; + std::shared_ptr seq_num_counter_; + std::shared_ptr path_factory_; + bool write_placeholder_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 2d63af11..53749b83 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -28,34 +28,46 @@ #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/object_utils.h" namespace paimon { CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader( std::unique_ptr&& reader, const std::optional& first_row_id, - int64_t snapshot_id, const std::shared_ptr& pool) + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool) : first_row_id_(first_row_id), snapshot_id_(snapshot_id), + file_field_names_(file_field_names), arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {} Status CompleteRowTrackingFieldsBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, - arrow::ImportSchema(c_file_schema.get())); + // The physical fields of the file decide which special fields must be stripped from the + // format reader's schema: a format without a self-describing file schema (e.g. blob) + // declares them via file_field_names_, self-describing formats are queried directly. + std::vector file_field_names; + if (file_field_names_) { + file_field_names = file_field_names_.value(); + } else { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, + reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); + file_field_names = file_schema->field_names(); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); read_schema_ = arrow_schema; int32_t row_id_idx = arrow_schema->GetFieldIndex(SpecialFields::RowId().Name()); - if (row_id_idx != -1 && file_schema->GetFieldIndex(SpecialFields::RowId().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + if (row_id_idx != -1 && + !ObjectUtils::Contains(file_field_names, SpecialFields::RowId().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(row_id_idx)); } int32_t sequence_id_idx = arrow_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); if (sequence_id_idx != -1 && - file_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()) == -1) { - // read special fields but file not exist, remove special fields to format reader + !ObjectUtils::Contains(file_field_names, SpecialFields::SequenceNumber().Name())) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(sequence_id_idx)); } ArrowSchema c_schema; @@ -88,15 +100,14 @@ CompleteRowTrackingFieldsBatchReader::NextBatchWithBitmap() { std::string row_id_field_name = SpecialFields::RowId().Name(); if (read_schema_->GetFieldIndex(row_id_field_name) != -1) { row_id_array = src_struct_array->GetFieldByName(row_id_field_name); - PAIMON_ASSIGN_OR_RAISE(uint64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); - auto row_id_convert_func = [previous_batch_first_row_number, - this](int32_t idx_in_array) -> Result { + auto row_id_convert_func = [this](int32_t idx_in_array) -> Result { if (first_row_id_ == std::nullopt) { return Status::Invalid( "unexpected: read _ROW_ID special field, but first row id is null in meta"); } - return first_row_id_.value() + previous_batch_first_row_number + idx_in_array; + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, + reader_->GetPreviousBatchFileRowId(idx_in_array)); + return first_row_id_.value() + file_row_id; }; PAIMON_RETURN_NOT_OK(ConvertRowTrackingField(src_struct_array->length(), /*init_value=*/0, row_id_convert_func, &row_id_array)); diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.h b/src/paimon/core/io/complete_row_tracking_fields_reader.h index 9c75c21e..7f4d7ead 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.h +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.h @@ -33,10 +33,13 @@ namespace paimon { // Precondition: read_schema has special fields class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { public: - CompleteRowTrackingFieldsBatchReader(std::unique_ptr&& reader, - const std::optional& first_row_id, - int64_t snapshot_id, - const std::shared_ptr& pool); + /// `file_field_names` declares the physical fields of a file whose format has no + /// self-describing schema (e.g. blob); when nullopt the file schema is queried from the + /// inner reader via GetFileSchema(). + CompleteRowTrackingFieldsBatchReader( + std::unique_ptr&& reader, const std::optional& first_row_id, + int64_t snapshot_id, const std::optional>& file_field_names, + const std::shared_ptr& pool); Result> GetFileSchema() const override { return Status::Invalid( @@ -62,8 +65,8 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { reader_->Close(); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -82,6 +85,7 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { private: std::optional first_row_id_; int64_t snapshot_id_ = -1; + std::optional> file_field_names_; std::shared_ptr arrow_pool_; std::shared_ptr read_schema_; std::unique_ptr reader_; diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp index 19b8e800..b6800c63 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp @@ -19,7 +19,10 @@ #include "paimon/core/io/complete_row_tracking_fields_reader.h" #include +#include +#include #include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -46,7 +49,8 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { std::make_unique(src_array, src_array->type(), batch_size); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), first_row_id, snapshot_id, pool_); + std::move(file_batch_reader), first_row_id, snapshot_id, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK(complete_row_tracking_fields_reader->SetReadSchema( @@ -65,12 +69,14 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test { void CheckSetReadSchema( const std::shared_ptr& file_schema, const std::shared_ptr& read_schema, - const std::shared_ptr& expected_schema_for_inner_reader) const { + const std::shared_ptr& expected_schema_for_inner_reader, + const std::optional>& file_field_names = std::nullopt) const { auto file_batch_reader = std::make_unique( /*data=*/nullptr, arrow::struct_(file_schema->fields()), /*read_batch_size=*/1); auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, pool_); + std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, + file_field_names, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -132,6 +138,33 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchema) { } } +TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchemaWithDeclaredFileFields) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + arrow::field("_ROW_ID", arrow::int64()), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + }; + { + // declared physical fields take precedence over the inner reader's self-described + // schema: the special fields are stripped even though the mock's file schema + // contains them + auto file_schema = arrow::schema(fields); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = arrow::schema({fields[0], fields[1]}); + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1"}); + } + { + // declared physical fields containing the special fields keep them for the inner reader + auto file_schema = arrow::schema({fields[0], fields[1]}); + auto read_schema = arrow::schema(fields); + auto expected_schema_for_inner_reader = read_schema; + CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader, + std::vector{"f0", "f1", "_ROW_ID", "_SEQUENCE_NUMBER"}); + } +} + TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestWithNoRowTrackingFields) { arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), @@ -344,7 +377,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidWithReadNonExistFiel auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( @@ -380,7 +414,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNextBatchBeforeSetRe auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ASSERT_NOK_WITH_MSG(complete_row_tracking_fields_reader->NextBatchWithBitmap(), "in CompleteRowTrackingFieldsBatchReader SetReadSchema is supposed to be " "called before NextBatch"); @@ -410,7 +445,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNullFirstRowId) { auto complete_row_tracking_fields_reader = std::make_shared( - std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, pool_); + std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, + /*file_field_names=*/std::nullopt, pool_); ArrowSchema c_read_schema; ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok()); ASSERT_OK( diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 293f14ab..12618a2e 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/fs/external_path_provider.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/stats/simple_stats.h" diff --git a/src/paimon/core/io/data_file_writer.cpp b/src/paimon/core/io/data_file_writer.cpp index 6f8bb3bb..4ed3e040 100644 --- a/src/paimon/core/io/data_file_writer.cpp +++ b/src/paimon/core/io/data_file_writer.cpp @@ -45,6 +45,10 @@ DataFileWriter::DataFileWriter( stats_extractor_(stats_extractor), write_cols_(write_cols) {} +void DataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) { + metadata_finalizer_ = std::move(finalizer); +} + Status DataFileWriter::Write(ArrowArray* batch) { int64_t record_count = batch->length; PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(batch)); @@ -52,6 +56,17 @@ Status DataFileWriter::Write(ArrowArray* batch) { return Status::OK(); } +Status DataFileWriter::BeforeFinish() { + if (metadata_finalizer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr updated_schema, + metadata_finalizer_()); + if (updated_schema) { + PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema)); + } + } + return Status::OK(); +} + Result> DataFileWriter::GetResult() { PAIMON_ASSIGN_OR_RAISE(std::vector> field_stats, GetFieldStats()); PAIMON_ASSIGN_OR_RAISE(SimpleStats stats, diff --git a/src/paimon/core/io/data_file_writer.h b/src/paimon/core/io/data_file_writer.h index 097c9b95..60cc808a 100644 --- a/src/paimon/core/io/data_file_writer.h +++ b/src/paimon/core/io/data_file_writer.h @@ -33,6 +33,10 @@ #include "paimon/result.h" #include "paimon/status.h" +namespace arrow { +class Schema; +} // namespace arrow + namespace paimon { class ColumnStats; @@ -42,6 +46,11 @@ class MemoryPool; class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr> { public: + /// Callback invoked during BeforeFinish() to finalize file metadata. + /// Produces an updated schema with per-field metadata (e.g. shredding metadata) + /// and may perform other finalization work (e.g. reporting stats to cross-file context). + using MetadataFinalizer = std::function>()>; + DataFileWriter(const std::string& compression, std::function converter, int64_t schema_id, const std::shared_ptr& seq_num_counter, FileSource file_source, @@ -49,14 +58,20 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr>& write_cols, const std::shared_ptr& pool); + /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated + /// schema and perform finalization callbacks. Must be set before Close(). + void SetMetadataFinalizer(MetadataFinalizer finalizer); + Status Write(::ArrowArray* batch) override; Result> GetResult() override; + protected: + Status BeforeFinish() override; + private: Result>> GetFieldStats(); - private: std::shared_ptr pool_; int64_t schema_id_; bool is_external_path_; @@ -65,6 +80,7 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr stats_extractor_; std::optional> write_cols_; + MetadataFinalizer metadata_finalizer_; }; } // namespace paimon diff --git a/src/paimon/core/io/data_file_writer_factory.cpp b/src/paimon/core/io/data_file_writer_factory.cpp new file mode 100644 index 00000000..b929dde8 --- /dev/null +++ b/src/paimon/core/io/data_file_writer_factory.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/data_file_writer_factory.h" + +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" +#include "paimon/format/file_format.h" +#include "paimon/format/writer_builder.h" + +namespace paimon { + +DataFileWriterFactory::DataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& pool) + : options_(options), schema_id_(schema_id), pool_(pool) {} + +Result DataFileWriterFactory::CreateWriterResources( + const FileFormat& format, const std::shared_ptr& file_schema, + bool create_stats_extractor) const { + WriterResources resources; + { + ::ArrowSchema arrow_schema; + ArrowSchemaMarkReleased(&arrow_schema); + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + resources.writer_builder, + format.CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); + resources.writer_builder->WithMemoryPool(pool_); + } + if (create_stats_extractor) { + ::ArrowSchema arrow_schema; + ArrowSchemaMarkReleased(&arrow_schema); + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE(resources.stats_extractor, + format.CreateStatsExtractor(&arrow_schema)); + } + return resources; +} + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_writer_factory.h b/src/paimon/core/io/data_file_writer_factory.h new file mode 100644 index 00000000..c727b47d --- /dev/null +++ b/src/paimon/core/io/data_file_writer_factory.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/core_options.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class FileFormat; +class FormatStatsExtractor; +class MemoryPool; +class WriterBuilder; + +class DataFileWriterFactory { + public: + DataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& pool); + virtual ~DataFileWriterFactory() = default; + + protected: + struct WriterResources { + std::shared_ptr writer_builder; + std::shared_ptr stats_extractor; + }; + + Result CreateWriterResources(const FileFormat& format, + const std::shared_ptr& file_schema, + bool create_stats_extractor) const; + + CoreOptions options_; + int64_t schema_id_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index 148b62db..3bd04b49 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include "arrow/api.h" @@ -37,48 +38,163 @@ #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" #include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/memory/bytes.h" #include "paimon/reader/batch_reader.h" namespace paimon { class MemoryPool; -FieldMappingReader::FieldMappingReader(int32_t field_count, - std::unique_ptr&& reader, - const BinaryRow& partition, - std::unique_ptr&& mapping, - const std::shared_ptr& pool) - : field_count_(field_count), - arrow_pool_(GetArrowPool(pool)), - reader_(std::move(reader)), - partition_(partition), - partition_info_(mapping->partition_info), - non_partition_info_(mapping->non_partition_info), - non_exist_field_info_(mapping->non_exist_field_info) { - if (non_exist_field_info_ != std::nullopt || partition_info_ != std::nullopt) { - need_mapping_ = true; +Result FieldMappingReader::HasMapSelectedKeysRecursively( + const std::shared_ptr& read_field) const { + if (!read_field) { + return false; + } + auto type_id = read_field->type()->id(); + if (type_id == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys, + NestedProjectionUtils::GetMapSelectedKeys(read_field)); + return !selected_keys.empty(); + } + if (type_id == arrow::Type::STRUCT) { + for (const auto& child : read_field->type()->fields()) { + PAIMON_ASSIGN_OR_RAISE(bool has_selected_keys, HasMapSelectedKeysRecursively(child)); + if (has_selected_keys) { + return true; + } + } + } + return false; +} + +Result> FieldMappingReader::FilterMapSelectedKeysRecursively( + const std::shared_ptr& array, + const std::shared_ptr& read_field) const { + if (!array || !read_field) { + return array; + } + + auto type_id = read_field->type()->id(); + if (type_id == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys, + NestedProjectionUtils::GetMapSelectedKeys(read_field)); + if (selected_keys.empty()) { + return array; + } + return NestedProjectionUtils::FilterMapArrayBySelectedKeys(array, selected_keys, + arrow_pool_.get()); + } + + if (type_id == arrow::Type::STRUCT) { + if (array->type_id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("FilterMapSelectedKeysRecursively requires struct array for read " + "field '{}', got {}", + read_field->name(), array->type()->ToString())); + } + auto struct_array = std::static_pointer_cast(array); + auto read_struct_type = std::static_pointer_cast(read_field->type()); + if (struct_array->num_fields() != read_struct_type->num_fields()) { + return Status::Invalid(fmt::format( + "FilterMapSelectedKeysRecursively struct field count mismatch for '{}': " + "array {} vs read {}", + read_field->name(), struct_array->num_fields(), read_struct_type->num_fields())); + } + + arrow::ArrayVector filtered_children; + std::vector> filtered_child_data; + filtered_children.reserve(struct_array->num_fields()); + filtered_child_data.reserve(struct_array->num_fields()); + for (int32_t i = 0; i < struct_array->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr filtered_child, + FilterMapSelectedKeysRecursively(struct_array->field(i), + read_struct_type->field(i))); + filtered_child_data.push_back(filtered_child->data()); + filtered_children.push_back(std::move(filtered_child)); + } + + // Preserve parent struct null semantics after filtering children. + auto filtered_struct_data = arrow::ArrayData::Make( + read_struct_type, struct_array->length(), {struct_array->null_bitmap()}, + std::move(filtered_child_data), struct_array->null_count(), struct_array->offset()); + return arrow::MakeArray(std::move(filtered_struct_data)); + } + + return array; +} + +Result> FieldMappingReader::Create( + int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, + std::unique_ptr&& mapping, + std::set&& skip_map_selected_keys_filter_field_ids, + const std::shared_ptr& pool) { + auto mapping_reader = std::unique_ptr(new FieldMappingReader( + field_count, std::move(reader), partition, std::move(mapping), pool)); + + mapping_reader->need_mapping_ = false; + mapping_reader->need_casting_ = false; + mapping_reader->skip_map_selected_keys_filter_field_ids_ = + std::move(skip_map_selected_keys_filter_field_ids); + + if (mapping_reader->non_exist_field_info_ != std::nullopt || + mapping_reader->partition_info_ != std::nullopt) { + mapping_reader->need_mapping_ = true; } for (int32_t i = 0; - i < static_cast(non_partition_info_.idx_in_target_read_schema.size()); i++) { - if (i != non_partition_info_.idx_in_target_read_schema[i]) { - need_mapping_ = true; + i < + static_cast(mapping_reader->non_partition_info_.idx_in_target_read_schema.size()); + i++) { + if (i != mapping_reader->non_partition_info_.idx_in_target_read_schema[i]) { + mapping_reader->need_mapping_ = true; } - if (non_partition_info_.cast_executors[i] != nullptr) { - need_casting_ = true; + if (mapping_reader->non_partition_info_.cast_executors[i] != nullptr) { + mapping_reader->need_casting_ = true; + } + // A differing nested type needs the AlignArrayToReadType reshape below. + if (!mapping_reader->non_partition_info_.non_partition_data_schema[i].Type()->Equals( + *mapping_reader->non_partition_info_.non_partition_read_schema[i].Type())) { + mapping_reader->need_casting_ = true; } // Field name change (RENAME COLUMN) also requires mapping: data schema // carries the file's physical name while read schema carries the // post-rename logical name. If we skipped mapping, the inner reader's // batch would be passed through with the old physical name and the // consumer's name-based lookup against the read schema would fail. - if (non_partition_info_.non_partition_data_schema[i].Name() != - non_partition_info_.non_partition_read_schema[i].Name()) { - need_mapping_ = true; + if (mapping_reader->non_partition_info_.non_partition_data_schema[i].Name() != + mapping_reader->non_partition_info_.non_partition_read_schema[i].Name()) { + mapping_reader->need_mapping_ = true; + } + // Map selected-keys metadata must be validated in Create() (fail-fast). + // Non-empty selected-keys also requires mapping so that + // FilterMapArrayBySelectedKeys can filter out unwanted entries. + PAIMON_ASSIGN_OR_RAISE( + bool has_map_selected_keys, + mapping_reader->HasMapSelectedKeysRecursively( + mapping_reader->non_partition_info_.non_partition_read_schema[i].ArrowField())); + if (has_map_selected_keys && + mapping_reader->skip_map_selected_keys_filter_field_ids_.count( + mapping_reader->non_partition_info_.non_partition_read_schema[i].Id()) == 0) { + mapping_reader->need_mapping_ = true; } } + + return mapping_reader; } +FieldMappingReader::FieldMappingReader(int32_t field_count, + std::unique_ptr&& reader, + const BinaryRow& partition, + std::unique_ptr&& mapping, + const std::shared_ptr& pool) + : field_count_(field_count), + arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + partition_(partition), + partition_info_(mapping->partition_info), + non_partition_info_(mapping->non_partition_info), + non_exist_field_info_(mapping->non_exist_field_info) {} + Result> FieldMappingReader::CastNonPartitionArrayIfNeed( const std::shared_ptr& src_array) const { if (!need_casting_) { @@ -92,6 +208,7 @@ Result> FieldMappingReader::CastNonPartitionArrayI casted_array.reserve(field_count); casted_field_names.reserve(field_count); for (int32_t i = 0; i < field_count; i++) { + std::shared_ptr column; if (non_partition_info_.cast_executors[i] != nullptr) { auto single_column_array = struct_array->field(i); // if src array is dict, cast to string first @@ -104,18 +221,27 @@ Result> FieldMappingReader::CastNonPartitionArrayI arrow::compute::CastOptions::Safe(), arrow_pool_.get())); } PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr casted, + column, non_partition_info_.cast_executors[i]->Cast( single_column_array, non_partition_info_.non_partition_read_schema[i].Type(), arrow_pool_.get())); - casted_array.push_back(casted); - casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); } else { // read and data type may both be string type, but after adapter transform, type may be // dictionary, need reconstruct struct type - casted_array.push_back(struct_array->field(i)); - casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); + column = struct_array->field(i); + } + // Null-fill nested fields added by schema evolution. Only when the data and + // read types differ -- the reader may hand back a dictionary-encoded array + // for an unchanged type, which is not a reshape target. + if (!non_partition_info_.non_partition_data_schema[i].Type()->Equals( + *non_partition_info_.non_partition_read_schema[i].Type())) { + PAIMON_ASSIGN_OR_RAISE( + column, NestedProjectionUtils::AlignArrayToReadType( + column, non_partition_info_.non_partition_read_schema[i].Type(), + arrow_pool_.get())); } + casted_array.push_back(column); + casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name()); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, arrow::StructArray::Make(casted_array, casted_field_names)); @@ -144,9 +270,9 @@ Result FieldMappingReader::NextBatchWithBitmap // mapping non-partition array PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_non_partition_array, CastNonPartitionArrayIfNeed(non_partition_array)); - MappingFields(casted_non_partition_array, non_partition_info_.non_partition_read_schema, - non_partition_info_.idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields( + casted_non_partition_array, non_partition_info_.non_partition_read_schema, + non_partition_info_.idx_in_target_read_schema, &target_array, &target_field_names)); // mapping partition array if (partition_info_ != std::nullopt) { @@ -155,9 +281,9 @@ Result FieldMappingReader::NextBatchWithBitmap GeneratePartitionArray(non_partition_array->length())); } auto trim_partition_array = partition_array_->Slice(0, non_partition_array->length()); - MappingFields(trim_partition_array, partition_info_.value().partition_read_schema, - partition_info_.value().idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields( + trim_partition_array, partition_info_.value().partition_read_schema, + partition_info_.value().idx_in_target_read_schema, &target_array, &target_field_names)); } // mapping non-exist array if (non_exist_field_info_ != std::nullopt) { @@ -166,9 +292,10 @@ Result FieldMappingReader::NextBatchWithBitmap GenerateNonExistArray(non_partition_array->length())); } auto trim_non_exist_array = non_exist_array_->Slice(0, non_partition_array->length()); - MappingFields(trim_non_exist_array, non_exist_field_info_.value().non_exist_read_schema, - non_exist_field_info_.value().idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields(trim_non_exist_array, + non_exist_field_info_.value().non_exist_read_schema, + non_exist_field_info_.value().idx_in_target_read_schema, + &target_array, &target_field_names)); } // construct target array @@ -285,20 +412,28 @@ Result> FieldMappingReader::GenerateNonExistArray( return arrow_array; } -void FieldMappingReader::MappingFields(const std::shared_ptr& data_array, - const std::vector& read_fields_of_data_array, - const std::vector& idx_in_target_schema, - arrow::ArrayVector* target_array, - std::vector* target_field_names) { +Status FieldMappingReader::MappingFields(const std::shared_ptr& data_array, + const std::vector& read_fields_of_data_array, + const std::vector& idx_in_target_schema, + arrow::ArrayVector* target_array, + std::vector* target_field_names) { auto* struct_array = arrow::internal::checked_cast(data_array.get()); assert(struct_array); assert(struct_array->fields().size() == idx_in_target_schema.size()); for (size_t i = 0; i < idx_in_target_schema.size(); i++) { - // target type may be string type, but after adapter transform, type may be dictionary, - // need reconstruct struct type - (*target_array)[idx_in_target_schema[i]] = struct_array->field(i); - (*target_field_names)[idx_in_target_schema[i]] = read_fields_of_data_array[i].Name(); + std::shared_ptr field_array = struct_array->field(i); + const DataField& read_field = read_fields_of_data_array[i]; + + // Filter map entries by selected keys recursively (supports MAP nested in STRUCT). + if (skip_map_selected_keys_filter_field_ids_.count(read_field.Id()) == 0) { + PAIMON_ASSIGN_OR_RAISE(field_array, FilterMapSelectedKeysRecursively( + field_array, read_field.ArrowField())); + } + + (*target_array)[idx_in_target_schema[i]] = std::move(field_array); + (*target_field_names)[idx_in_target_schema[i]] = read_field.Name(); } + return Status::OK(); } } // namespace paimon diff --git a/src/paimon/core/io/field_mapping_reader.h b/src/paimon/core/io/field_mapping_reader.h index a5aad35b..7a5edc16 100644 --- a/src/paimon/core/io/field_mapping_reader.h +++ b/src/paimon/core/io/field_mapping_reader.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -48,9 +49,11 @@ struct FieldMapping; class FieldMappingReader : public FileBatchReader { public: - FieldMappingReader(int32_t field_count, std::unique_ptr&& reader, - const BinaryRow& partition, std::unique_ptr&& mapping, - const std::shared_ptr& pool); + static Result> Create( + int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, + std::unique_ptr&& mapping, + std::set&& skip_map_selected_keys_filter_field_ids, + const std::shared_ptr& pool); Result NextBatch() override { return Status::Invalid( @@ -76,8 +79,8 @@ class FieldMappingReader : public FileBatchReader { return Status::Invalid("FieldMappingReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -89,6 +92,10 @@ class FieldMappingReader : public FileBatchReader { } private: + FieldMappingReader(int32_t field_count, std::unique_ptr&& reader, + const BinaryRow& partition, std::unique_ptr&& mapping, + const std::shared_ptr& pool); + Result> GenerateSinglePartitionArray(int32_t idx, int32_t batch_size) const; @@ -98,11 +105,18 @@ class FieldMappingReader : public FileBatchReader { Result> CastNonPartitionArrayIfNeed( const std::shared_ptr& src_array) const; - static void MappingFields(const std::shared_ptr& src_array, - const std::vector& read_fields_of_data_array, - const std::vector& idx_in_target_schema, - arrow::ArrayVector* target_array, - std::vector* target_field_names); + Status MappingFields(const std::shared_ptr& src_array, + const std::vector& read_fields_of_data_array, + const std::vector& idx_in_target_schema, + arrow::ArrayVector* target_array, + std::vector* target_field_names); + + Result HasMapSelectedKeysRecursively( + const std::shared_ptr& read_field) const; + + Result> FilterMapSelectedKeysRecursively( + const std::shared_ptr& array, + const std::shared_ptr& read_field) const; private: bool need_mapping_ = false; @@ -115,6 +129,7 @@ class FieldMappingReader : public FileBatchReader { std::optional partition_info_; NonPartitionInfo non_partition_info_; std::optional non_exist_field_info_; + std::set skip_map_selected_keys_filter_field_ids_; std::shared_ptr partition_array_; std::shared_ptr non_exist_array_; diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 4b64a0b0..233a8587 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -34,6 +34,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/util/checked_cast.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/defs.h" @@ -46,6 +47,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -133,9 +135,11 @@ class FieldMappingReaderTest : public ::testing::Test { mapping->non_partition_info.non_partition_filter, /*batch_size=*/1); - auto reader = std::make_shared( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition_, - std::move(mapping), pool_); + ASSERT_OK_AND_ASSIGN( + auto reader, + FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition_, + std::move(mapping), /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -172,9 +176,11 @@ class FieldMappingReaderTest : public ::testing::Test { data_path, fs, data_schema.get(), data_array, arrow_schema.get(), /*predicate=*/mapping->non_partition_info.non_partition_filter, /*batch_size=*/1); - auto reader = std::make_shared( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition, - std::move(mapping), pool_); + ASSERT_OK_AND_ASSIGN( + auto reader, + FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition, + std::move(mapping), /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -183,6 +189,9 @@ class FieldMappingReaderTest : public ::testing::Test { auto expected_chunk_array = std::make_shared(arrow::ArrayVector({expect_array})); + ASSERT_TRUE(result_array->type()->Equals(expected_chunk_array->type())) + << result_array->type()->ToString() << expected_chunk_array->type()->ToString(); + ASSERT_TRUE(result_array->Equals(expected_chunk_array)) << result_array->ToString() << expected_chunk_array->ToString(); } @@ -229,8 +238,11 @@ TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { {false, static_cast(1), static_cast(2), static_cast(3), static_cast(4), std::string("5"), std::make_shared("6", pool_.get()), 100}, pool_.get()); - auto mapping_reader = std::make_unique( - /*field_count=*/8, /*reader=*/nullptr, partition, std::move(field_mapping), pool_); + ASSERT_OK_AND_ASSIGN( + auto mapping_reader, + FieldMappingReader::Create( + /*field_count=*/8, /*reader=*/nullptr, partition, std::move(field_mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); { ASSERT_OK_AND_ASSIGN(auto p7_array, mapping_reader->GenerateSinglePartitionArray( @@ -438,6 +450,91 @@ TEST_F(FieldMappingReaderTest, TestDictionaryTypeWithSchemaEvolution) { partition, expected_array); } +TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) { + // A field `c`(id=12) was added inside the list's struct after the file was + // written. Reading the old file with the new schema must null-fill `c`. + auto id_field = [](const std::string& name, const std::shared_ptr& type, + int32_t id) { + return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type))); + }; + auto data_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11)}); + auto read_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11), + id_field("c", arrow::int32(), 12)}); + std::vector data_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))}; + std::vector read_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))}; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([ + [[[1, "x"], [2, "y"]]], + [[[3, "z"]]] + ])") + .ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, + /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields)); + auto mock = std::make_unique( + data_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8); + ASSERT_OK_AND_ASSIGN(auto reader, FieldMappingReader::Create( + read_schema->num_fields(), std::move(mock), + BinaryRow::EmptyRow(), std::move(mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); + + auto expect_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([ + [[[1, "x", null], [2, "y", null]]], + [[[3, "z", null]]] + ])") + .ValueOrDie(); + auto expected_chunk = std::make_shared(arrow::ArrayVector({expect_array})); + ASSERT_TRUE(result_array->type()->Equals(expected_chunk->type())) + << result_array->type()->ToString() << " vs " << expected_chunk->type()->ToString(); + ASSERT_TRUE(result_array->Equals(expected_chunk)) + << result_array->ToString() << " vs " << expected_chunk->ToString(); +} + +TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideListOrc) { + // ORC round-trip: added field inside a list's struct is null-filled. + auto id_field = [](const std::string& name, const std::shared_ptr& type, + int32_t id) { + return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type))); + }; + auto data_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11)}); + auto read_struct = + arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11), + id_field("c", arrow::int32(), 12)}); + std::vector data_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))}; + std::vector read_fields = { + DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))}; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([ + [[[1, 2], [3, 4]]], + [[[5, 6]]] + ])") + .ValueOrDie()); + auto expect_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([ + [[[1, 2, null], [3, 4, null]]], + [[[5, 6, null]]] + ])") + .ValueOrDie(); + CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, /*partition_keys=*/{}, + BinaryRow::EmptyRow(), expect_array); +} + TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithModifyType) { std::vector data_fields = {DataField(0, arrow::field("f0", arrow::utf8())), DataField(1, arrow::field("f1", arrow::float32())), @@ -689,6 +786,33 @@ TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithDictType) { partition, expected_array); } +TEST_F(FieldMappingReaderTest, TestReadInlineBlobAsBinaryDataFile) { + // data_fields uses binary type because inline blob fields are stored as binary in data files + std::vector data_fields = { + DataField(0, arrow::field("descriptor", arrow::binary(), /*nullable=*/true)), + }; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + std::string json_str = R"([ + ["descriptor-1"], + [null], + ["descriptor-2"] + ])"; + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), json_str) + .ValueOrDie()); + + std::vector read_fields = { + DataField(0, BlobUtils::ToArrowField("descriptor", /*nullable=*/true)), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), json_str) + .ValueOrDie()); + + CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, + /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); +} + TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { // Test all 4 combinations of rename × cast: // f0: no rename, no cast (utf8 → utf8, name unchanged) @@ -731,4 +855,27 @@ TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); } + +TEST_F(FieldMappingReaderTest, TestCreateFailFastOnInvalidMapSelectedKeysMetadata) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,a"}); + auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), + /*nullable=*/true, metadata); + + NonPartitionInfo non_part_info; + non_part_info.non_partition_data_schema = {DataField(0, map_field)}; + non_part_info.non_partition_read_schema = {DataField(0, map_field)}; + non_part_info.idx_in_target_read_schema = {0}; + non_part_info.cast_executors = {nullptr}; + + auto field_mapping = std::make_unique( + /*partition_info=*/PartitionInfo(), std::move(non_part_info), + /*non_exist_field_info=*/std::nullopt); + + ASSERT_NOK_WITH_MSG(FieldMappingReader::Create( + /*field_count=*/1, + /*reader=*/nullptr, + /*partition=*/BinaryRow::EmptyRow(), std::move(field_mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_), + "Duplicate selected key 'a'"); +} } // namespace paimon::test diff --git a/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp b/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp new file mode 100644 index 00000000..253db982 --- /dev/null +++ b/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/generic_row_to_arrow_array_converter.h" + +#include +#include +#include +#include + +#include "arrow/array/builder_nested.h" +#include "arrow/memory_pool.h" +#include "arrow/util/checked_cast.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> GenericRowToArrowArrayConverter::Create( + const std::shared_ptr& schema, arrow::MemoryPool* pool) { + std::unique_ptr array_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder( + pool, std::make_shared(schema->fields()), &array_builder)); + + auto struct_builder = + arrow::internal::checked_pointer_cast(std::move(array_builder)); + assert(struct_builder); + std::vector appenders; + appenders.reserve(schema->num_fields()); + int32_t reserve_count = 1; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE( + RowToArrowArrayConverter::AppendValueFunc func, + AppendField(/*use_view=*/true, struct_builder->field_builder(i), &reserve_count)); + appenders.emplace_back(std::move(func)); + } + return std::unique_ptr(new GenericRowToArrowArrayConverter( + reserve_count, std::move(appenders), std::move(struct_builder), nullptr)); +} + +Result GenericRowToArrowArrayConverter::NextBatch( + const std::vector& rows) { + PAIMON_RETURN_NOT_OK(ResetAndReserve()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + array_builder_->AppendValues(rows.size(), /*valid_bytes=*/nullptr)); + for (size_t i = 0; i < appenders_.size(); ++i) { + for (const auto& row : rows) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(appenders_[i](row, i)); + } + } + + return FinishAndAccumulate(); +} + +} // namespace paimon diff --git a/src/paimon/core/io/generic_row_to_arrow_array_converter.h b/src/paimon/core/io/generic_row_to_arrow_array_converter.h new file mode 100644 index 00000000..77df6c8f --- /dev/null +++ b/src/paimon/core/io/generic_row_to_arrow_array_converter.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/generic_row.h" +#include "paimon/core/io/row_to_arrow_array_converter.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" + +namespace arrow { +class MemoryPool; +class StructBuilder; +} // namespace arrow + +namespace paimon { + +/// Converts in-memory GenericRow values into a struct Arrow array. +class GenericRowToArrowArrayConverter + : public RowToArrowArrayConverter { + public: + static Result> Create( + const std::shared_ptr& schema, arrow::MemoryPool* pool); + + Result NextBatch(const std::vector& rows) override; + + private: + GenericRowToArrowArrayConverter(int32_t reserve_count, std::vector&& appenders, + std::unique_ptr&& array_builder, + std::unique_ptr&& arrow_pool) + : RowToArrowArrayConverter(reserve_count, std::move(appenders), std::move(array_builder), + std::move(arrow_pool)) {} +}; + +} // namespace paimon diff --git a/src/paimon/core/io/infer_shredding_file_writer.h b/src/paimon/core/io/infer_shredding_file_writer.h new file mode 100644 index 00000000..4acf9b48 --- /dev/null +++ b/src/paimon/core/io/infer_shredding_file_writer.h @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/core/io/single_file_writer.h" +#include "paimon/core/key_value.h" + +namespace paimon { + +/// A file writer that infers the shredding write plan from the first rows of the file. Incoming +/// batches are buffered until `ShreddingWritePlanFactory::InferBufferRowCount` rows have been +/// collected (or the writer is closed); the buffered batches are then sampled to create the +/// batch converter, the actual file writer is created with the resulting physical schema, and +/// the buffered batches are replayed into it. File-size rolling is suppressed while buffering; +/// row-count rolling remains controlled by the outer RollingFileWriter. +template +class InferShreddingFileWriter : public SingleFileWriter { + public: + using CreateInnerFn = std::function>>( + const std::shared_ptr&)>; + + InferShreddingFileWriter(const std::shared_ptr& logical_schema, + const std::shared_ptr& plan_factory, + const std::string& file_format_identifier, CreateInnerFn create_inner) + : SingleFileWriter(/*compression=*/"", std::function()), + logical_type_(arrow::struct_(logical_schema->fields())), + plan_factory_(plan_factory), + file_format_identifier_(file_format_identifier), + create_inner_(std::move(create_inner)) {} + + Status Write(T record) override { + if (plan_finalized_) { + return inner_->Write(std::move(record)); + } + PAIMON_RETURN_NOT_OK(Buffer(std::move(record))); + if (buffered_rows_ >= plan_factory_->InferBufferRowCount()) { + return FinalizePlanAndFlush(); + } + return Status::OK(); + } + + Status Close() override { + if (!plan_finalized_) { + PAIMON_RETURN_NOT_OK(FinalizePlanAndFlush()); + } + return inner_->Close(); + } + + Result GetResult() override { + if (!inner_) { + return Status::Invalid("Cannot access the result unless the writer is closed."); + } + return inner_->GetResult(); + } + + Result ReachTargetSize(bool suggested_check, int64_t target_size) override { + if (!plan_finalized_) { + // File-size rolling is unavailable until the inner writer has been created. + return false; + } + return inner_->ReachTargetSize(suggested_check, target_size); + } + + Result::AbortExecutor> GetAbortExecutor() const override { + if (!inner_) { + return Status::Invalid("Writer should be closed!"); + } + return inner_->GetAbortExecutor(); + } + + std::string GetPath() const override { + return inner_ ? inner_->GetPath() : ""; + } + + void Abort() override { + if (inner_) { + inner_->Abort(); + } + } + + int64_t RecordCount() const override { + return plan_finalized_ ? inner_->RecordCount() : buffered_rows_; + } + + std::shared_ptr GetMetrics() const override { + return inner_ ? inner_->GetMetrics() : nullptr; + } + + private: + struct BufferedBatch { + std::shared_ptr batch; + // Holds the non-batch part of a KeyValueBatch record; unused for plain batches. + KeyValueBatch record_template; + }; + + Status Buffer(T record) { + BufferedBatch buffered; + ::ArrowArray* c_batch; + if constexpr (std::is_same_v) { + c_batch = record; + } else { + static_assert(std::is_same_v, + "InferShreddingFileWriter supports ::ArrowArray* and KeyValueBatch"); + buffered.record_template = std::move(record); + c_batch = buffered.record_template.batch.get(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(buffered.batch, + arrow::ImportArray(c_batch, logical_type_)); + buffered_rows_ += buffered.batch->length(); + buffered_batches_.push_back(std::move(buffered)); + return Status::OK(); + } + + Status FinalizePlanAndFlush() { + std::vector> samples; + samples.reserve(buffered_batches_.size()); + for (const auto& buffered : buffered_batches_) { + samples.push_back(buffered.batch); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + plan_factory_->CreateConverter(file_format_identifier_, samples)); + PAIMON_ASSIGN_OR_RAISE(inner_, create_inner_(converter)); + plan_finalized_ = true; + for (auto& buffered : buffered_batches_) { + if constexpr (std::is_same_v) { + ::ArrowArray c_batch; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*buffered.batch, &c_batch)); + PAIMON_RETURN_NOT_OK(inner_->Write(&c_batch)); + } else { + KeyValueBatch record = std::move(buffered.record_template); + record.batch = std::make_unique<::ArrowArray>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*buffered.batch, record.batch.get())); + PAIMON_RETURN_NOT_OK(inner_->Write(std::move(record))); + } + } + buffered_batches_.clear(); + return Status::OK(); + } + + std::shared_ptr logical_type_; + std::shared_ptr plan_factory_; + std::string file_format_identifier_; + CreateInnerFn create_inner_; + + std::vector buffered_batches_; + int64_t buffered_rows_ = 0; + bool plan_finalized_ = false; + std::unique_ptr> inner_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/infer_shredding_file_writer_test.cpp b/src/paimon/core/io/infer_shredding_file_writer_test.cpp new file mode 100644 index 00000000..b0e82756 --- /dev/null +++ b/src/paimon/core/io/infer_shredding_file_writer_test.cpp @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/infer_shredding_file_writer.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +namespace { + +/// A file-less writer standing in for the actual data file writer: it applies the shredding +/// conversion like the injected converter lambda would and collects the written batches. +class CollectingFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr> { + public: + CollectingFileWriter(const std::shared_ptr& converter, + const std::shared_ptr& logical_type, + std::vector>* sink) + : SingleFileWriter("", std::function()), + converter_(converter), + logical_type_(logical_type), + sink_(sink) {} + + Status Write(::ArrowArray* record) override { + record_count_ += record->length; + std::shared_ptr array; + if (converter_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, + converter_->Convert(record)); + auto physical_type = arrow::struct_(converter_->GetPhysicalSchema()->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array, + arrow::ImportArray(physical.get(), physical_type)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array, arrow::ImportArray(record, logical_type_)); + } + sink_->push_back(std::move(array)); + return Status::OK(); + } + + Status Close() override { + closed = true; + return Status::OK(); + } + + Result> GetResult() override { + return std::shared_ptr(nullptr); + } + + Result ReachTargetSize(bool suggested_check, int64_t target_size) override { + return false; + } + + Result GetAbortExecutor() const override { + return AbortExecutor(nullptr, ""); + } + + void Abort() override {} + + int64_t RecordCount() const override { + return record_count_; + } + + bool closed = false; + + private: + std::shared_ptr converter_; + std::shared_ptr logical_type_; + std::vector>* sink_; + int64_t record_count_ = 0; +}; + +} // namespace + +class InferShreddingFileWriterTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + logical_type_ = arrow::struct_(schema_->fields()); + } + + std::shared_ptr BuildBatch(const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN( + std::shared_ptr batch, + VariantTestData::BuildVariantBatch(schema_->field(0), schema_->field(1), jsons, pool_)); + return batch; + } + + Status WriteBatch(InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>* w, + const std::vector& jsons) { + std::shared_ptr array = BuildBatch(jsons); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + return w->Write(&c_array); + } + + std::unique_ptr>> + MakeWriter(int32_t buffer_rows) { + std::map option_map = { + {"variant.inferShreddingSchema", "true"}, + {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)}, + // Keep the manifest format resolvable in test binaries without the avro plugin. + {"manifest.format", "parquet"}}; + EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map)); + auto plan_factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + auto create_inner = [this](const std::shared_ptr& converter) + -> Result< + std::unique_ptr>>> { + captured_converters_.push_back(converter); + auto writer = std::make_unique(converter, logical_type_, &sink_); + inner_ = writer.get(); + return std::unique_ptr>>( + std::move(writer)); + }; + return std::make_unique< + InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>>( + schema_, plan_factory, "parquet", create_inner); + } + + protected: + std::shared_ptr pool_; + std::shared_ptr schema_; + std::shared_ptr logical_type_; + // The converters own the arrow pool backing the collected arrays; keep them declared first + // so the arrays are destroyed before the pool. + std::vector> captured_converters_; + std::vector> sink_; + CollectingFileWriter* inner_ = nullptr; +}; + +TEST_F(InferShreddingFileWriterTest, BuffersUntilThresholdThenReplays) { + auto writer = MakeWriter(/*buffer_rows=*/4); + // File-size rolling is suppressed while buffering. + ASSERT_OK_AND_ASSIGN(bool reach, writer->ReachTargetSize(true, 1)); + ASSERT_FALSE(reach); + + ASSERT_OK(WriteBatch( + writer.get(), {R"({"age": 35, "city": "Chicago"})", R"({"age": 25, "city": "Hangzhou"})"})); + ASSERT_TRUE(sink_.empty()); + ASSERT_EQ(writer->RecordCount(), 2); + + // Crossing the row threshold finalizes the plan and replays the buffered batches. + ASSERT_OK(WriteBatch( + writer.get(), {R"({"age": 18, "city": "Beijing"})", R"({"age": 60, "city": "Shanghai"})"})); + ASSERT_EQ(sink_.size(), 2); + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_NE(captured_converters_[0], nullptr); + const auto& physical_type = static_cast(*sink_[0]->type()); + const auto& variant_physical = + static_cast(*physical_type.GetFieldByName("v")->type()); + ASSERT_NE(variant_physical.GetFieldByName("typed_value"), nullptr); + + // Subsequent writes stream through the finalized writer directly. + ASSERT_OK(WriteBatch(writer.get(), {R"({"age": 1, "city": "Suzhou"})"})); + ASSERT_EQ(sink_.size(), 3); + ASSERT_EQ(writer->RecordCount(), 5); + + ASSERT_OK(writer->Close()); + ASSERT_TRUE(inner_->closed); +} + +TEST_F(InferShreddingFileWriterTest, CloseFlushesPartialBuffer) { + auto writer = MakeWriter(/*buffer_rows=*/100); + ASSERT_OK(WriteBatch(writer.get(), {R"({"age": 35, "city": "Chicago"})"})); + ASSERT_TRUE(sink_.empty()); + ASSERT_OK(writer->Close()); + ASSERT_EQ(sink_.size(), 1); + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_NE(captured_converters_[0], nullptr); + ASSERT_TRUE(inner_->closed); +} + +TEST_F(InferShreddingFileWriterTest, EmptyFileUsesUntypedVariantPlan) { + auto writer = MakeWriter(/*buffer_rows=*/4); + ASSERT_OK(writer->Close()); + // Even without typed fields, inference creates the complete untyped Variant physical plan. + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_NE(captured_converters_[0], nullptr); + const auto& physical_type = static_cast( + *captured_converters_[0]->GetPhysicalSchema()->GetFieldByName("v")->type()); + ASSERT_EQ(physical_type.GetFieldByName("typed_value"), nullptr); + ASSERT_TRUE(sink_.empty()); + ASSERT_TRUE(inner_->closed); +} + +} // namespace paimon::test diff --git a/src/paimon/core/io/key_value_data_file_record_reader.cpp b/src/paimon/core/io/key_value_data_file_record_reader.cpp index bccd730f..18e27679 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.cpp +++ b/src/paimon/core/io/key_value_data_file_record_reader.cpp @@ -52,11 +52,11 @@ KeyValueDataFileRecordReader::KeyValueDataFileRecordReader( Result KeyValueDataFileRecordReader::Iterator::HasNext() const { int64_t array_length = reader_->row_kind_array_->length(); - const auto& selection_bitmap = reader_->selection_bitmap_; - if (selection_bitmap.Cardinality() == array_length) { + if (selection_cardinality_ == array_length) { // all rows are selected in bitmap return cursor_ < array_length; } + const auto& selection_bitmap = reader_->selection_bitmap_; auto iter = selection_bitmap.EqualOrLarger(cursor_); if (iter == selection_bitmap.End()) { // no row are selected @@ -70,7 +70,7 @@ Result KeyValueDataFileRecordReader::Iterator::HasNext() const { Result KeyValueDataFileRecordReader::Iterator::Next() { // key is only used in merge sort; key context does not hold parent struct array - auto key = std::make_unique(reader_->key_ctx_, cursor_); + std::shared_ptr key = std::make_shared(reader_->key_ctx_, cursor_); // value is used in merge sort and projection (maybe async and multi-thread), so value context // holds parent struct array to ensure data remains valid auto value = std::make_unique(reader_->value_ctx_, cursor_); @@ -83,15 +83,15 @@ Result KeyValueDataFileRecordReader::Iterator::Next() { Result> KeyValueDataFileRecordReader::Iterator::NextWithFilePos() { PAIMON_ASSIGN_OR_RAISE(KeyValue kv, Next()); - return std::make_pair(previous_batch_first_row_number_ + cursor_ - 1, std::move(kv)); + PAIMON_ASSIGN_OR_RAISE(uint64_t global_row_id, + reader_->reader_->GetPreviousBatchFileRowId(cursor_ - 1)); + return std::make_pair(static_cast(global_row_id), std::move(kv)); } Result> KeyValueDataFileRecordReader::NextBatch() { Reset(); PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap()); - PAIMON_ASSIGN_OR_RAISE(int64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); if (BatchReader::IsEofBatch(batch_with_bitmap)) { // reader eof, just return return std::unique_ptr(); @@ -142,8 +142,7 @@ Result> KeyValueDataFileRecordRe key_ctx_ = std::make_shared(key_fields, pool_); value_ctx_ = std::make_shared(value_fields, pool_); ArrowUtils::TraverseArray(data_batch); - return std::make_unique( - this, previous_batch_first_row_number); + return std::make_unique(this); } void KeyValueDataFileRecordReader::Reset() { diff --git a/src/paimon/core/io/key_value_data_file_record_reader.h b/src/paimon/core/io/key_value_data_file_record_reader.h index 5cd1f589..1ed08257 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.h +++ b/src/paimon/core/io/key_value_data_file_record_reader.h @@ -56,16 +56,16 @@ class KeyValueDataFileRecordReader : public KeyValueRecordReader { class Iterator : public KeyValueRecordReader::Iterator { public: - Iterator(KeyValueDataFileRecordReader* reader, int64_t previous_batch_first_row_number) - : previous_batch_first_row_number_(previous_batch_first_row_number), reader_(reader) {} + explicit Iterator(KeyValueDataFileRecordReader* reader) + : reader_(reader), selection_cardinality_(reader->selection_bitmap_.Cardinality()) {} Result HasNext() const override; Result Next() override; Result> NextWithFilePos(); private: - int64_t previous_batch_first_row_number_; mutable int64_t cursor_ = 0; KeyValueDataFileRecordReader* reader_ = nullptr; + int64_t selection_cardinality_ = 0; }; Result> NextBatch() override; diff --git a/src/paimon/core/io/key_value_data_file_writer.cpp b/src/paimon/core/io/key_value_data_file_writer.cpp index 82bbba92..9393c7c3 100644 --- a/src/paimon/core/io/key_value_data_file_writer.cpp +++ b/src/paimon/core/io/key_value_data_file_writer.cpp @@ -64,6 +64,10 @@ KeyValueDataFileWriter::KeyValueDataFileWriter( is_external_path_(is_external_path), disable_stats_(stats_extractor == nullptr) {} +void KeyValueDataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) { + metadata_finalizer_ = std::move(finalizer); +} + Status KeyValueDataFileWriter::Write(KeyValueBatch batch) { // update min and max key if (!min_key_) { @@ -75,10 +79,22 @@ Status KeyValueDataFileWriter::Write(KeyValueBatch batch) { max_sequence_number_ = std::max(max_sequence_number_, batch.max_sequence_number); // update delete row count delete_row_count_ += batch.delete_row_count; + PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(std::move(batch))); return Status::OK(); } +Status KeyValueDataFileWriter::BeforeFinish() { + if (metadata_finalizer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr updated_schema, + metadata_finalizer_()); + if (updated_schema) { + PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema)); + } + } + return Status::OK(); +} + Result> KeyValueDataFileWriter::GetResult() { PAIMON_ASSIGN_OR_RAISE(std::vector> field_stats, GetFieldStats()); if (!disable_stats_ && field_stats.size() != static_cast(write_schema_->num_fields())) { diff --git a/src/paimon/core/io/key_value_data_file_writer.h b/src/paimon/core/io/key_value_data_file_writer.h index f62bf64c..e1e3fd92 100644 --- a/src/paimon/core/io/key_value_data_file_writer.h +++ b/src/paimon/core/io/key_value_data_file_writer.h @@ -47,6 +47,11 @@ class SimpleStats; class KeyValueDataFileWriter : public SingleFileWriter> { public: + /// Callback invoked during BeforeFinish() to finalize file metadata. + /// Produces an updated schema with per-field metadata (e.g. shredding metadata) + /// and may perform other finalization work (e.g. reporting stats to cross-file context). + using MetadataFinalizer = std::function>()>; + KeyValueDataFileWriter(const std::string& compression, std::function converter, int64_t schema_id, int32_t level, FileSource file_source, @@ -55,10 +60,17 @@ class KeyValueDataFileWriter const std::shared_ptr& write_schema, bool is_external_path, const std::shared_ptr& pool); + /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated + /// schema and perform finalization callbacks. Must be set before Close(). + void SetMetadataFinalizer(MetadataFinalizer finalizer); + Status Write(KeyValueBatch batch) override; Result> GetResult() override; + protected: + Status BeforeFinish() override; + private: Result>> GetFieldStats(); @@ -84,6 +96,7 @@ class KeyValueDataFileWriter int64_t max_sequence_number_ = std::numeric_limits::min(); std::shared_ptr min_key_; std::shared_ptr max_key_; + MetadataFinalizer metadata_finalizer_; }; } // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp new file mode 100644 index 00000000..07d50b98 --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/key_value_data_file_writer_factory.h" + +#include +#include + +#include "arrow/c/helpers.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/key_value_data_file_writer.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +KeyValueDataFileWriterFactory::KeyValueDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + write_schema_(write_schema), + level_(level), + file_source_(file_source), + primary_keys_(primary_keys), + path_factory_(path_factory), + create_stats_extractor_(create_stats_extractor) {} + +Result>>> +KeyValueDataFileWriterFactory::CreateWriter() const { + std::function converter = + [](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { + ArrowArrayMove(key_value_batch.batch.get(), array); + return Status::OK(); + }; + + auto format = options_.GetWriteFileFormat(level_); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*format, write_schema_, create_stats_extractor_)); + auto writer = std::make_unique( + options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_, + file_source_, primary_keys_, resources.stats_extractor, write_schema_, + path_factory_->IsExternalPath(), pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.h b/src/paimon/core/io/key_value_data_file_writer_factory.h new file mode 100644 index 00000000..6ac50aba --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factory.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/key_value.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class MemoryPool; + +class KeyValueDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory> { + public: + KeyValueDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, + FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, + bool create_stats_extractor, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + protected: + std::shared_ptr write_schema_; + int32_t level_; + FileSource file_source_; + std::vector primary_keys_; + std::shared_ptr path_factory_; + bool create_stats_extractor_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_in_memory_record_reader.cpp b/src/paimon/core/io/key_value_in_memory_record_reader.cpp index 65aa5cba..43af1b54 100644 --- a/src/paimon/core/io/key_value_in_memory_record_reader.cpp +++ b/src/paimon/core/io/key_value_in_memory_record_reader.cpp @@ -47,7 +47,7 @@ Result KeyValueInMemoryRecordReader::Iterator::Next() { } // key must hold value_struct_array as min/max key may be used after projection - auto key = std::make_unique(reader_->key_ctx_, index); + std::shared_ptr key = std::make_shared(reader_->key_ctx_, index); auto value = std::make_unique(reader_->value_ctx_, index); return KeyValue(row_kind, reader_->last_sequence_num_ + index, /*level=*/KeyValue::UNKNOWN_LEVEL, std::move(key), std::move(value)); diff --git a/src/paimon/core/io/rolling_blob_file_writer.cpp b/src/paimon/core/io/rolling_blob_file_writer.cpp index ccd6ca45..36336eef 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.cpp +++ b/src/paimon/core/io/rolling_blob_file_writer.cpp @@ -18,6 +18,7 @@ #include "paimon/core/io/rolling_blob_file_writer.h" +#include #include #include #include @@ -29,7 +30,6 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "fmt/format.h" -#include "fmt/ranges.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" @@ -42,22 +42,23 @@ class DataType; namespace paimon { RollingBlobFileWriter::RollingBlobFileWriter( - int64_t target_file_size, - std::function>()> create_file_writer, + int64_t target_file_size, int64_t target_file_row_num, + const std::shared_ptr& writer_factory, const std::shared_ptr& blob_schema, MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator, - const std::shared_ptr& data_type) - : RollingFileWriter<::ArrowArray*, std::shared_ptr>(target_file_size, - create_file_writer), + const std::shared_ptr& data_type, const std::set& inline_fields) + : RollingFileWriter<::ArrowArray*, std::shared_ptr>( + target_file_size, target_file_row_num, writer_factory), blob_schema_(blob_schema), blob_writer_creator_(std::move(blob_writer_creator)), data_type_(data_type), + inline_fields_(inline_fields), logger_(Logger::GetLogger("RollingBlobFileWriter")) {} Status RollingBlobFileWriter::Write(::ArrowArray* record) { ScopeGuard guard([this]() -> void { this->Abort(); }); // Open the current writer if write the first record or roll over happen before. - if (PAIMON_UNLIKELY(current_writer_ == nullptr)) { + if (writer_factory_ != nullptr && PAIMON_UNLIKELY(current_writer_ == nullptr)) { PAIMON_RETURN_NOT_OK(OpenCurrentWriter()); } if (PAIMON_UNLIKELY(blob_writer_ == nullptr)) { @@ -69,14 +70,16 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { auto struct_array = std::dynamic_pointer_cast(arrow_array); PAIMON_ASSIGN_OR_RAISE(BlobUtils::SeparatedStructArrays separated_arrays, - BlobUtils::SeparateBlobArray(struct_array)); + BlobUtils::SeparateBlobArray(struct_array, inline_fields_)); // Write main (non-blob) data - ::ArrowArray c_main_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - arrow::ExportArray(*separated_arrays.main_array, &c_main_array)); - ScopeGuard array_lifecycle_guard( - [&c_main_array]() -> void { ArrowArrayRelease(&c_main_array); }); - PAIMON_RETURN_NOT_OK(current_writer_->Write(&c_main_array)); + if (current_writer_ != nullptr) { + ::ArrowArray c_main_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*separated_arrays.main_array, &c_main_array)); + ScopeGuard array_lifecycle_guard( + [&c_main_array]() -> void { ArrowArrayRelease(&c_main_array); }); + PAIMON_RETURN_NOT_OK(current_writer_->Write(&c_main_array)); + } // Write blob data via MultipleBlobFileWriter (each blob field independently) ::ArrowArray c_blob_array; @@ -86,7 +89,12 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { PAIMON_RETURN_NOT_OK(blob_writer_->Write(&c_blob_array)); record_count_ += record_count; - PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile()); + current_file_record_count_ += record_count; + bool need_rolling_file = current_file_record_count_ >= target_file_row_num_; + if (!need_rolling_file && current_writer_ != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool main_writer_needs_rolling, NeedRollingFile()); + need_rolling_file = main_writer_needs_rolling; + } if (need_rolling_file) { PAIMON_RETURN_NOT_OK(CloseCurrentWriter()); } @@ -95,22 +103,24 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { } Status RollingBlobFileWriter::CloseCurrentWriter() { - if (current_writer_ == nullptr) { - return Status::OK(); - } if (blob_writer_ == nullptr) { return Status::OK(); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr main_data_file_meta, CloseMainWriter()); + std::shared_ptr main_data_file_meta; + if (current_writer_ != nullptr) { + PAIMON_ASSIGN_OR_RAISE(main_data_file_meta, CloseMainWriter()); + } PAIMON_ASSIGN_OR_RAISE(std::vector> blob_metas, CloseBlobWriter()); - PAIMON_RETURN_NOT_OK( - ValidateFileConsistency(main_data_file_meta, blob_metas, blob_schema_->num_fields())); - results_.push_back(main_data_file_meta); + if (main_data_file_meta != nullptr) { + PAIMON_RETURN_NOT_OK(ValidateFileConsistency(main_data_file_meta, blob_metas)); + results_.push_back(main_data_file_meta); + } results_.insert(results_.end(), blob_metas.begin(), blob_metas.end()); current_writer_.reset(); + current_file_record_count_ = 0; return Status::OK(); } @@ -123,7 +133,8 @@ Result>> RollingBlobFileWriter::GetRes Result> RollingBlobFileWriter::CloseMainWriter() { PAIMON_RETURN_NOT_OK(current_writer_->Close()); - PAIMON_ASSIGN_OR_RAISE(auto abort_executor, current_writer_->GetAbortExecutor()); + PAIMON_ASSIGN_OR_RAISE(MainWriter::AbortExecutor abort_executor, + current_writer_->GetAbortExecutor()); closed_writers_.push_back(abort_executor); return current_writer_->GetResult(); } @@ -138,29 +149,25 @@ Result>> RollingBlobFileWriter::CloseB Status RollingBlobFileWriter::ValidateFileConsistency( const std::shared_ptr& main_data_file_meta, - const std::vector>& blob_tagged_metas, int32_t blob_field_count) { - if (blob_tagged_metas.empty()) { - return Status::OK(); - } - // With multiple blob fields, each blob field produces its own set of files. - // total_blob_row_count should be exactly main_row_count * blob_field_count. - int64_t main_row_count = main_data_file_meta->row_count; - int64_t expected_blob_row_count = main_row_count * blob_field_count; - int64_t total_blob_row_count = 0; + const std::vector>& blob_tagged_metas) { + std::map blob_field_row_counts; for (const auto& blob_tagged_meta : blob_tagged_metas) { - total_blob_row_count += blob_tagged_meta->row_count; + if (!blob_tagged_meta->write_cols || blob_tagged_meta->write_cols->empty()) { + return Status::Invalid( + fmt::format("This is a bug: Blob file {} must contain a write column.", + blob_tagged_meta->file_name)); + } + blob_field_row_counts[blob_tagged_meta->write_cols->at(0)] += blob_tagged_meta->row_count; } - if (total_blob_row_count != expected_blob_row_count) { - std::vector blob_file_names; - for (const auto& blob_tagged_meta : blob_tagged_metas) { - blob_file_names.push_back(blob_tagged_meta->file_name); + + int64_t main_row_count = main_data_file_meta->row_count; + for (const auto& [field_name, row_count] : blob_field_row_counts) { + if (row_count != main_row_count) { + return Status::Invalid(fmt::format( + "This is a bug: The row count of main file and blob file does not match. Main " + "file: {} (row count: {}), blob field name: {} (row count: {})", + main_data_file_meta->file_name, main_row_count, field_name, row_count)); } - return Status::Invalid(fmt::format( - "This is a bug: The row count of main file and blob files does not match. " - "Main file: {} (row count: {}), blob field count: {}, " - "expected blob row count: {}, blob files: {} (actual total row count: {})", - main_data_file_meta->file_name, main_row_count, blob_field_count, - expected_blob_row_count, fmt::join(blob_file_names, ", "), total_blob_row_count)); } return Status::OK(); } diff --git a/src/paimon/core/io/rolling_blob_file_writer.h b/src/paimon/core/io/rolling_blob_file_writer.h index f35f6960..4f3e6020 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.h +++ b/src/paimon/core/io/rolling_blob_file_writer.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/array/array_nested.h" @@ -39,7 +40,8 @@ namespace paimon { /// between them. /// /// Multiple blob fields are supported. Each blob field is written to its own set of blob files -/// independently via MultipleBlobFileWriter. +/// independently via MultipleBlobFileWriter. For blob-only writes, the main writer factory may be +/// nullptr and only blob files are produced. /// /// 
 /// For example,
@@ -59,12 +61,14 @@ class RollingBlobFileWriter
     : public RollingFileWriter<::ArrowArray*, std::shared_ptr> {
  public:
     using MainWriter = SingleFileWriter<::ArrowArray*, std::shared_ptr>;
+    using MainWriterFactory = SingleFileWriterFactory<::ArrowArray*, std::shared_ptr>;
 
-    RollingBlobFileWriter(int64_t target_file_size,
-                          std::function>()> create_file_writer,
+    RollingBlobFileWriter(int64_t target_file_size, int64_t target_file_row_num,
+                          const std::shared_ptr& writer_factory,
                           const std::shared_ptr& blob_schema,
                           MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator,
-                          const std::shared_ptr& data_type);
+                          const std::shared_ptr& data_type,
+                          const std::set& inline_fields);
     ~RollingBlobFileWriter() override = default;
 
     Status Write(::ArrowArray* record) override;
@@ -75,8 +79,7 @@ class RollingBlobFileWriter
  private:
     static Status ValidateFileConsistency(
         const std::shared_ptr& main_data_file_meta,
-        const std::vector>& blob_tagged_metas,
-        int32_t blob_field_count);
+        const std::vector>& blob_tagged_metas);
 
     Status CloseCurrentWriter();
 
@@ -87,6 +90,7 @@ class RollingBlobFileWriter
     MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator_;
     std::unique_ptr blob_writer_;
     std::shared_ptr data_type_;
+    std::set inline_fields_;
 
     std::unique_ptr logger_;
 };
diff --git a/src/paimon/core/io/rolling_blob_file_writer_test.cpp b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
index 9e2a4be5..654c9126 100644
--- a/src/paimon/core/io/rolling_blob_file_writer_test.cpp
+++ b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
@@ -84,11 +84,15 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3,
         /*write_cols=*/std::vector({"blob"}));
-    ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3},
-                                                             /*blob_field_count=*/1));
-    ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2},
-                                                                       /*blob_field_count=*/2),
-                        "This is a bug: The row count of main file and blob files does not match.");
+    ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3}));
+    ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2}),
+                        "This is a bug: The row count of main file and blob file does not match.");
+
+    file_meta2->write_cols = std::vector({"blob1"});
+    file_meta3->write_cols = std::vector({"blob2"});
+    ASSERT_NOK_WITH_MSG(
+        RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3}),
+        "This is a bug: The row count of main file and blob file does not match.");
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/rolling_file_writer.h b/src/paimon/core/io/rolling_file_writer.h
index fa8a51df..3a945632 100644
--- a/src/paimon/core/io/rolling_file_writer.h
+++ b/src/paimon/core/io/rolling_file_writer.h
@@ -18,14 +18,16 @@
 
 #pragma once
 
+#include 
 #include 
-#include 
+#include 
 #include 
 
 #include "arrow/c/bridge.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/core/io/file_writer.h"
 #include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/single_file_writer_factory.h"
 #include "paimon/core/key_value.h"
 #include "paimon/metrics.h"
 #include "paimon/record_batch.h"
@@ -36,11 +38,11 @@ namespace paimon {
 template 
 class RollingFileWriter : public FileWriter> {
  public:
-    RollingFileWriter(
-        int64_t target_file_size,
-        std::function>>()> create_file_writer)
+    RollingFileWriter(int64_t target_file_size, int64_t target_file_row_num,
+                      const std::shared_ptr>& writer_factory)
         : target_file_size_(target_file_size),
-          create_file_writer(create_file_writer),
+          target_file_row_num_(target_file_row_num),
+          writer_factory_(writer_factory),
           metrics_(std::make_shared()),
           logger_(Logger::GetLogger("RollingFileWriter")) {}
 
@@ -59,10 +61,6 @@ class RollingFileWriter : public FileWriter> {
         return metrics_;
     }
 
-    int64_t TargetFileSize() const {
-        return target_file_size_;
-    }
-
  protected:
     static constexpr int32_t CHECK_ROLLING_RECORD_CNT = 1000;
 
@@ -72,10 +70,12 @@ class RollingFileWriter : public FileWriter> {
     Status OpenCurrentWriter();
 
     int64_t target_file_size_ = 0;
-    std::function>>()> create_file_writer;
+    int64_t target_file_row_num_ = std::numeric_limits::max();
+    std::shared_ptr> writer_factory_;
     std::shared_ptr metrics_;
 
     int64_t record_count_ = 0;
+    int64_t current_file_record_count_ = 0;
     int64_t last_need_rolling_record_count_ = 0;
     bool closed_ = false;
 
@@ -101,6 +101,9 @@ bool RollingFileWriter::SuggestCheck() {
 
 template 
 Result RollingFileWriter::NeedRollingFile() {
+    if (current_file_record_count_ >= target_file_row_num_) {
+        return true;
+    }
     return current_writer_->ReachTargetSize(SuggestCheck(), target_file_size_);
 }
 
@@ -121,6 +124,7 @@ Status RollingFileWriter::Write(T record) {
     }
     PAIMON_RETURN_NOT_OK(current_writer_->Write(std::move(record)));
     record_count_ += record_count;
+    current_file_record_count_ += record_count;
     PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile());
     if (need_rolling_file) {
         PAIMON_RETURN_NOT_OK(CloseCurrentWriter());
@@ -139,7 +143,7 @@ Result> RollingFileWriter::GetResult() {
 
 template 
 Result>> RollingFileWriter::NewWriter() {
-    return create_file_writer();
+    return writer_factory_->CreateWriter();
 }
 
 template 
@@ -156,13 +160,16 @@ Status RollingFileWriter::CloseCurrentWriter() {
     if (current_writer_ == nullptr) {
         return Status::OK();
     }
-    std::shared_ptr current_metrics = current_writer_->GetMetrics();
     PAIMON_RETURN_NOT_OK(current_writer_->Close());
+    // Read the metrics after Close(): writers that create their inner writer lazily (e.g.
+    // inferred shredding) only expose metrics once closed.
+    std::shared_ptr current_metrics = current_writer_->GetMetrics();
     PAIMON_ASSIGN_OR_RAISE(auto abort_executor, current_writer_->GetAbortExecutor());
     closed_writers_.push_back(abort_executor);
     PAIMON_ASSIGN_OR_RAISE(R result, current_writer_->GetResult());
     results_.push_back(result);
     current_writer_.reset();
+    current_file_record_count_ = 0;
     if (metrics_) {
         metrics_->Merge(current_metrics);
     }
diff --git a/src/paimon/core/io/row_to_arrow_array_converter.h b/src/paimon/core/io/row_to_arrow_array_converter.h
index 247a315b..43c97afd 100644
--- a/src/paimon/core/io/row_to_arrow_array_converter.h
+++ b/src/paimon/core/io/row_to_arrow_array_converter.h
@@ -131,7 +131,7 @@ Status RowToArrowArrayConverter::Reserve(arrow::ArrayBuilder* array_builde
         case arrow::Type::type::FLOAT:
         case arrow::Type::type::DOUBLE:
         case arrow::Type::type::TIMESTAMP:
-        case arrow::Type::type::DECIMAL:
+        case arrow::Type::type::DECIMAL128:
             break;
         case arrow::Type::type::STRING: {
             // reserve string data buffer
@@ -203,7 +203,7 @@ Status RowToArrowArrayConverter::Accumulate(const arrow::Array* array, int
         case arrow::Type::type::FLOAT:
         case arrow::Type::type::DOUBLE:
         case arrow::Type::type::TIMESTAMP:
-        case arrow::Type::type::DECIMAL:
+        case arrow::Type::type::DECIMAL128:
             break;
         case arrow::Type::type::STRING: {
             auto string_array = arrow::internal::checked_cast(array);
@@ -405,7 +405,7 @@ RowToArrowArrayConverter::AppendField(bool use_view, arrow::ArrayBuilder*
                         DateTimeUtils::TimestampToInteger(timestamp, time_type));
                 });
         }
-        case arrow::Type::type::DECIMAL: {
+        case arrow::Type::type::DECIMAL128: {
             PAIMON_ASSIGN_OR_RAISE(auto* field_builder,
                                    CastToTypedBuilder(array_builder));
             auto decimal_type =
diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
new file mode 100644
index 00000000..6e4843bb
--- /dev/null
+++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/shredding_append_data_file_writer_factory.h"
+
+#include 
+
+#include "arrow/c/helpers.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/data_file_writer.h"
+#include "paimon/core/io/infer_shredding_file_writer.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+ShreddingAppendDataFileWriterFactory::ShreddingAppendDataFileWriterFactory(
+    const CoreOptions& options, int64_t schema_id,
+    const std::shared_ptr& write_schema,
+    const std::optional>& write_cols,
+    const std::shared_ptr& seq_num_counter, FileSource file_source,
+    const std::shared_ptr& path_factory,
+    const std::shared_ptr& plan_factory,
+    const std::shared_ptr& pool)
+    : AppendDataFileWriterFactory(options, schema_id, write_schema, write_cols, seq_num_counter,
+                                  file_source, path_factory, pool),
+      plan_factory_(plan_factory) {}
+
+Result>>>
+ShreddingAppendDataFileWriterFactory::CreateWriter() const {
+    if (!plan_factory_) {
+        return Status::Invalid("Shredding append writer requires a write-plan factory.");
+    }
+    const std::string format_identifier = options_.GetFileFormat()->Identifier();
+    if (plan_factory_->ShouldInferWritePlan()) {
+        auto create_inner = [this](const std::shared_ptr& converter) {
+            return CreateShreddedWriter(converter);
+        };
+        return std::make_unique<
+            InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>>(
+            write_schema_, plan_factory_, format_identifier, std::move(create_inner));
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr converter,
+        plan_factory_->CreateConverter(format_identifier, /*sample_batches=*/{}));
+    return CreateShreddedWriter(converter);
+}
+
+Result>>>
+ShreddingAppendDataFileWriterFactory::CreateShreddedWriter(
+    const std::shared_ptr& converter) const {
+    if (converter == nullptr) {
+        // No conversion is useful for this file; fall back to the plain writer.
+        std::unique_ptr>> writer;
+        PAIMON_ASSIGN_OR_RAISE(writer, AppendDataFileWriterFactory::CreateWriter());
+        writer->SetCompletionCallback(
+            [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+        return writer;
+    }
+    std::shared_ptr seq_num_counter = ResolveSeqNumCounter();
+    std::shared_ptr file_schema = converter->GetPhysicalSchema();
+    std::function batch_converter =
+        [converter](::ArrowArray* input, ::ArrowArray* output) -> Status {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, converter->Convert(input));
+        ArrowArrayMove(physical.get(), output);
+        return Status::OK();
+    };
+    PAIMON_ASSIGN_OR_RAISE(WriterResources resources,
+                           CreateWriterResources(*options_.GetFileFormat(), file_schema,
+                                                 /*create_stats_extractor=*/true));
+    auto writer = std::make_unique(
+        options_.GetFileCompression(), std::move(batch_converter), schema_id_, seq_num_counter,
+        file_source_, resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_,
+        pool_);
+    PAIMON_RETURN_NOT_OK(
+        writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
+    ShreddingWritePlanFactory::MetadataFinalizer finalizer =
+        plan_factory_->CreateMetadataFinalizer(converter, options_.GetFileCompression());
+    if (finalizer) {
+        writer->SetMetadataFinalizer(std::move(finalizer));
+    }
+    writer->SetCompletionCallback(
+        [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+    return std::unique_ptr>>(
+        std::move(writer));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.h b/src/paimon/core/io/shredding_append_data_file_writer_factory.h
new file mode 100644
index 00000000..a69d9e5d
--- /dev/null
+++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.h
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/shredding/shredding_write_plan_factory.h"
+#include "paimon/core/io/append_data_file_writer_factory.h"
+
+namespace paimon {
+
+class DataFilePathFactory;
+class LongCounter;
+class MemoryPool;
+
+/// Creates append data file writers that rewrite logical batches into a physical (shredded)
+/// layout as planned by a `ShreddingWritePlanFactory` (MAP shared-shredding or VARIANT
+/// shredding, configured or inferred).
+class ShreddingAppendDataFileWriterFactory : public AppendDataFileWriterFactory {
+ public:
+    ShreddingAppendDataFileWriterFactory(
+        const CoreOptions& options, int64_t schema_id,
+        const std::shared_ptr& write_schema,
+        const std::optional>& write_cols,
+        const std::shared_ptr& seq_num_counter, FileSource file_source,
+        const std::shared_ptr& path_factory,
+        const std::shared_ptr& plan_factory,
+        const std::shared_ptr& pool);
+
+    Result>>>
+    CreateWriter() const override;
+
+ private:
+    Result>>>
+    CreateShreddedWriter(const std::shared_ptr& converter) const;
+
+    std::shared_ptr plan_factory_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
new file mode 100644
index 00000000..8ac583ee
--- /dev/null
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
+
+#include 
+
+#include "arrow/c/helpers.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/infer_shredding_file_writer.h"
+#include "paimon/core/io/key_value_data_file_writer.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory(
+    const CoreOptions& options, int64_t schema_id,
+    const std::shared_ptr& write_schema, int32_t level, FileSource file_source,
+    const std::vector& primary_keys,
+    const std::shared_ptr& path_factory, bool create_stats_extractor,
+    const std::shared_ptr& plan_factory,
+    const std::shared_ptr& pool)
+    : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source,
+                                    primary_keys, path_factory, create_stats_extractor, pool),
+      plan_factory_(plan_factory) {}
+
+Result>>>
+ShreddingKeyValueDataFileWriterFactory::CreateWriter() const {
+    if (!plan_factory_) {
+        return Status::Invalid("Shredding key-value writer requires a write-plan factory.");
+    }
+    const std::string format_identifier = options_.GetWriteFileFormat(level_)->Identifier();
+    if (plan_factory_->ShouldInferWritePlan()) {
+        auto create_inner = [this](const std::shared_ptr& converter) {
+            return CreateShreddedWriter(converter);
+        };
+        return std::make_unique<
+            InferShreddingFileWriter>>(
+            write_schema_, plan_factory_, format_identifier, std::move(create_inner));
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr converter,
+        plan_factory_->CreateConverter(format_identifier, /*sample_batches=*/{}));
+    return CreateShreddedWriter(converter);
+}
+
+Result>>>
+ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
+    const std::shared_ptr& converter) const {
+    if (converter == nullptr) {
+        // No conversion is useful for this file; fall back to the plain writer.
+        std::unique_ptr>> writer;
+        PAIMON_ASSIGN_OR_RAISE(writer, KeyValueDataFileWriterFactory::CreateWriter());
+        writer->SetCompletionCallback(
+            [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+        return writer;
+    }
+    auto format = options_.GetWriteFileFormat(level_);
+    std::shared_ptr file_schema = converter->GetPhysicalSchema();
+    std::function batch_converter =
+        [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical,
+                               converter->Convert(key_value_batch.batch.get()));
+        ArrowArrayMove(physical.get(), array);
+        return Status::OK();
+    };
+    PAIMON_ASSIGN_OR_RAISE(WriterResources resources,
+                           CreateWriterResources(*format, file_schema, create_stats_extractor_));
+    auto writer = std::make_unique(
+        options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_,
+        file_source_, primary_keys_, resources.stats_extractor, file_schema,
+        path_factory_->IsExternalPath(), pool_);
+    PAIMON_RETURN_NOT_OK(
+        writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
+    ShreddingWritePlanFactory::MetadataFinalizer finalizer =
+        plan_factory_->CreateMetadataFinalizer(converter, options_.GetWriteFileCompression(level_));
+    if (finalizer) {
+        writer->SetMetadataFinalizer(std::move(finalizer));
+    }
+    writer->SetCompletionCallback(
+        [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+    return std::unique_ptr>>(
+        std::move(writer));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h
new file mode 100644
index 00000000..fb967588
--- /dev/null
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/shredding/shredding_write_plan_factory.h"
+#include "paimon/core/io/key_value_data_file_writer_factory.h"
+
+namespace paimon {
+
+class DataFilePathFactory;
+class MemoryPool;
+
+/// Creates key-value data file writers that rewrite logical batches into a physical (shredded)
+/// layout as planned by a `ShreddingWritePlanFactory` (MAP shared-shredding or VARIANT
+/// shredding, configured or inferred).
+class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFactory {
+ public:
+    ShreddingKeyValueDataFileWriterFactory(
+        const CoreOptions& options, int64_t schema_id,
+        const std::shared_ptr& write_schema, int32_t level, FileSource file_source,
+        const std::vector& primary_keys,
+        const std::shared_ptr& path_factory, bool create_stats_extractor,
+        const std::shared_ptr& plan_factory,
+        const std::shared_ptr& pool);
+
+    Result>>>
+    CreateWriter() const override;
+
+ private:
+    Result>>>
+    CreateShreddedWriter(const std::shared_ptr& converter) const;
+
+    std::shared_ptr plan_factory_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/single_file_writer.h b/src/paimon/core/io/single_file_writer.h
index 2ead8aec..224e8d9e 100644
--- a/src/paimon/core/io/single_file_writer.h
+++ b/src/paimon/core/io/single_file_writer.h
@@ -21,12 +21,15 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
 
 #include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "arrow/ipc/api.h"
 #include "fmt/format.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/scope_guard.h"
@@ -41,6 +44,10 @@
 #include "paimon/result.h"
 #include "paimon/status.h"
 
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
 namespace paimon {
 
 class RecordBatch;
@@ -92,6 +99,12 @@ class SingleFileWriter : public FileWriter {
     void Abort() override;
     Status Close() override;
 
+    /// Sets a callback invoked only after the format writer and output stream have both
+    /// completed successfully.
+    void SetCompletionCallback(std::function callback) {
+        completion_callback_ = std::move(callback);
+    }
+
     std::shared_ptr GetMetrics() const override {
         if (writer_) {
             return writer_->GetWriterMetrics();
@@ -99,20 +112,29 @@ class SingleFileWriter : public FileWriter {
         return nullptr;
     }
 
-    Result ReachTargetSize(bool suggested_check, int64_t target_size);
+    virtual Result ReachTargetSize(bool suggested_check, int64_t target_size);
 
-    Result GetAbortExecutor() const {
+    virtual Result GetAbortExecutor() const {
         if (closed_ == false) {
             return Status::Invalid("Writer should be closed!");
         }
         return AbortExecutor(fs_, path_);
     }
 
-    std::string GetPath() const {
+    virtual std::string GetPath() const {
         return path_;
     }
 
  protected:
+    /// Hook called after Flush() and before Finish() during Close().
+    /// Subclasses can override to update per-field metadata before the file is finalized.
+    virtual Status BeforeFinish() {
+        return Status::OK();
+    }
+
+    /// Serializes schema and forwards it as file metadata to FormatWriter.
+    Status UpdateSchema(const std::shared_ptr& schema);
+
     int64_t output_bytes_ = -1;
     std::string compression_;
     std::function converter_;
@@ -120,6 +142,7 @@ class SingleFileWriter : public FileWriter {
     std::shared_ptr out_;  // nullptr for DirectWriterBuilder
     bool closed_ = false;
     std::string path_;
+    std::function completion_callback_;
 
  private:
     int64_t record_count_ = 0;
@@ -201,16 +224,24 @@ Status SingleFileWriter::Close() {
                         path_.c_str());
     });
     PAIMON_RETURN_NOT_OK(writer_->Flush());
+    PAIMON_RETURN_NOT_OK(BeforeFinish());
     PAIMON_RETURN_NOT_OK(writer_->Finish());
     if (out_) {
         PAIMON_RETURN_NOT_OK(out_->Flush());
         PAIMON_ASSIGN_OR_RAISE(output_bytes_, out_->GetPos());
-        PAIMON_RETURN_NOT_OK(out_->Close());
+        std::shared_ptr out = std::move(out_);
+        PAIMON_RETURN_NOT_OK(out->Close());
     } else {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(path_));
         output_bytes_ = file_status->GetLen();
     }
+    // Completing the format writer and stream is terminal even if publication fails. The scope
+    // guard still removes the file on a callback error, while a repeated Close() does not publish
+    // the same file again.
     closed_ = true;
+    if (completion_callback_) {
+        PAIMON_RETURN_NOT_OK(completion_callback_());
+    }
     guard.Release();
     return Status::OK();
 }
@@ -220,10 +251,25 @@ Result SingleFileWriter::ReachTargetSize(bool suggested_check, int64
     return writer_->ReachTargetSize(suggested_check, target_size);
 }
 
+template 
+Status SingleFileWriter::UpdateSchema(const std::shared_ptr& schema) {
+    if (!writer_) {
+        return Status::Invalid("Cannot update schema: format writer is not initialized.");
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr serialized,
+                                      arrow::ipc::SerializeSchema(*schema));
+    std::map metadata;
+    metadata.emplace(ArrowUtils::kArrowSchemaMetadataKey,
+                     std::string(reinterpret_cast(serialized->data()),
+                                 static_cast(serialized->size())));
+    return writer_->AddMetadata(metadata);
+}
+
 template 
 void SingleFileWriter::Abort() {
     if (out_) {
-        auto status = out_->Close();
+        std::shared_ptr out = std::move(out_);
+        auto status = out->Close();
         if (!status.ok()) {
             PAIMON_LOG_WARN(logger_, "Exception occurs when closing %s: %s", path_.c_str(),
                             status.ToString().c_str());
diff --git a/src/paimon/core/io/single_file_writer_factory.h b/src/paimon/core/io/single_file_writer_factory.h
new file mode 100644
index 00000000..8ec8ecf0
--- /dev/null
+++ b/src/paimon/core/io/single_file_writer_factory.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+
+#include "paimon/core/io/single_file_writer.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+template 
+class SingleFileWriterFactory {
+ public:
+    virtual ~SingleFileWriterFactory() = default;
+
+    virtual Result>> CreateWriter() const = 0;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index c3ef9eb0..8ed940e2 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -106,4 +106,42 @@ TEST(SingleFileWriterTest, TestInvalidConvert) {
     ASSERT_FALSE(exist);
 }
 
+TEST(SingleFileWriterTest, CompletionCallbackFailureIsTerminal) {
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    std::string file_path = dir->Str() + "/single-file";
+    auto data_type = arrow::struct_({arrow::field("col", arrow::int32())});
+    auto converter = [&](int32_t value, ::ArrowArray* dest) -> Status {
+        std::string value_str = "[[" + std::to_string(value) + "]]";
+        auto array =
+            arrow::ipc::internal::json::ArrayFromJSON(data_type, value_str.c_str()).ValueOrDie();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, dest));
+        return Status::OK();
+    };
+    SimpleSingleFileWriter writer("zstd", converter);
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}}));
+    auto file_format = options.GetWriteFileFormat(/*level=*/0);
+    auto file_system = options.GetFileSystem();
+    ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportType(*data_type, &arrow_schema).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer_builder,
+                         file_format->CreateWriterBuilder(&arrow_schema, /*batch_size=*/100));
+    ASSERT_OK(writer.Init(file_system, file_path, writer_builder));
+    ASSERT_OK(writer.Write(100));
+
+    int32_t callback_count = 0;
+    writer.SetCompletionCallback([&callback_count]() -> Status {
+        ++callback_count;
+        return Status::Invalid("completion failed");
+    });
+    ASSERT_NOK_WITH_MSG(writer.Close(), "completion failed");
+    ASSERT_EQ(callback_count, 1);
+    ASSERT_OK(writer.Close());
+    ASSERT_EQ(callback_count, 1);
+    ASSERT_OK_AND_ASSIGN(auto exists, file_system->Exists(file_path));
+    ASSERT_FALSE(exists);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/manifest/index_manifest_file.cpp b/src/paimon/core/manifest/index_manifest_file.cpp
index fda878c1..648c7302 100644
--- a/src/paimon/core/manifest/index_manifest_file.cpp
+++ b/src/paimon/core/manifest/index_manifest_file.cpp
@@ -63,7 +63,7 @@ Result> IndexManifestFile::Create(
         path_factory->CreateIndexManifestFileFactory();
     return std::unique_ptr(
         new IndexManifestFile(file_system, reader_builder, writer_builder, compression,
-                              index_manifest_file_factory, bucket_mode, pool));
+                              index_manifest_file_factory, bucket_mode, options.GetCache(), pool));
 }
 
 IndexManifestFile::IndexManifestFile(const std::shared_ptr& file_system,
@@ -71,10 +71,11 @@ IndexManifestFile::IndexManifestFile(const std::shared_ptr& file_sys
                                      const std::shared_ptr& writer_builder,
                                      const std::string& compression,
                                      const std::shared_ptr& path_factory,
-                                     int32_t bucket_mode, const std::shared_ptr& pool)
+                                     int32_t bucket_mode, const std::shared_ptr& cache,
+                                     const std::shared_ptr& pool)
     : ObjectsFile(file_system, reader_builder, writer_builder,
                                       std::make_unique(pool),
-                                      compression, path_factory, pool),
+                                      compression, path_factory, cache, pool),
       bucket_mode_(bucket_mode) {}
 
 Result> IndexManifestFile::WriteIndexFiles(
diff --git a/src/paimon/core/manifest/index_manifest_file.h b/src/paimon/core/manifest/index_manifest_file.h
index e98e0923..f6fd46f3 100644
--- a/src/paimon/core/manifest/index_manifest_file.h
+++ b/src/paimon/core/manifest/index_manifest_file.h
@@ -33,6 +33,7 @@
 
 namespace paimon {
 class CoreOptions;
+class Cache;
 class FileFormat;
 class FileStorePathFactory;
 class FileSystem;
@@ -62,7 +63,7 @@ class IndexManifestFile : public ObjectsFile {
                       const std::shared_ptr& writer_builder,
                       const std::string& compression,
                       const std::shared_ptr& path_factory, int32_t bucket_mode,
-                      const std::shared_ptr& pool);
+                      const std::shared_ptr& cache, const std::shared_ptr& pool);
 
     const int32_t bucket_mode_;
 };
diff --git a/src/paimon/core/manifest/index_manifest_file_handler.cpp b/src/paimon/core/manifest/index_manifest_file_handler.cpp
index efa6b610..2fb1ceb4 100644
--- a/src/paimon/core/manifest/index_manifest_file_handler.cpp
+++ b/src/paimon/core/manifest/index_manifest_file_handler.cpp
@@ -32,33 +32,31 @@ std::vector IndexManifestFileHandler::BucketedCombiner::Comb
     const std::vector& new_index_files) const {
     std::unordered_map index_entries;
     for (const auto& entry : prev_index_files) {
-        index_entries.emplace(
+        index_entries.insert_or_assign(
             BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), entry);
     }
 
-    std::unordered_map removed;
+    std::vector removed;
     removed.reserve(new_index_files.size());
-    std::unordered_map added;
+    std::vector added;
     added.reserve(new_index_files.size());
 
     for (const auto& entry : new_index_files) {
         if (entry.kind == FileKind::Delete()) {
-            removed.emplace(
-                BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()),
-                entry);
+            removed.push_back(entry);
         } else if (entry.kind == FileKind::Add()) {
-            added.emplace(
-                BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()),
-                entry);
+            added.push_back(entry);
         }
     }
 
     // The deleted entry is processed first to avoid overwriting a new entry.
     for (const auto& entry : removed) {
-        index_entries.erase(entry.first);
+        index_entries.erase(
+            BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()));
     }
     for (const auto& entry : added) {
-        index_entries.emplace(entry.first, entry.second);
+        index_entries.insert_or_assign(
+            BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), entry);
     }
 
     std::vector result_entries;
@@ -74,7 +72,7 @@ std::vector IndexManifestFileHandler::GlobalFileNameCombiner
     const std::vector& new_index_files) const {
     std::map index_entries;
     for (const auto& entry : prev_index_files) {
-        index_entries.emplace(entry.index_file->FileName(), entry);
+        index_entries.insert_or_assign(entry.index_file->FileName(), entry);
     }
 
     std::vector removed;
@@ -95,7 +93,7 @@ std::vector IndexManifestFileHandler::GlobalFileNameCombiner
         index_entries.erase(entry.index_file->FileName());
     }
     for (const auto& entry : added) {
-        index_entries.emplace(entry.index_file->FileName(), entry);
+        index_entries.insert_or_assign(entry.index_file->FileName(), entry);
     }
 
     std::vector result_entries;
diff --git a/src/paimon/core/manifest/index_manifest_file_handler.h b/src/paimon/core/manifest/index_manifest_file_handler.h
index 05dd8576..910b70f5 100644
--- a/src/paimon/core/manifest/index_manifest_file_handler.h
+++ b/src/paimon/core/manifest/index_manifest_file_handler.h
@@ -48,7 +48,7 @@ class IndexManifestFileHandler {
             const std::vector& new_index_files) const = 0;
     };
 
-    /// Combine previous and new global index files by file `BucketIdentifier`.
+    /// Combine previous and new index files by partition, bucket and index type.
     class BucketedCombiner : public IndexManifestFileCombiner {
      public:
         std::vector Combine(
@@ -56,7 +56,7 @@ class IndexManifestFileHandler {
             const std::vector& new_index_files) const override;
     };
 
-    /// Combine previous and new global index files by file name.
+    /// Combine previous and new index files by file name.
     class GlobalFileNameCombiner : public IndexManifestFileCombiner {
      public:
         std::vector Combine(
diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
index 15d1e9c4..2265e18b 100644
--- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
+++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp
@@ -76,6 +76,25 @@ class IndexManifestFileHandlerTest : public testing::Test {
                                             /*external_path=*/std::nullopt));
     }
 
+    static IndexManifestEntry MakeDvEntry(const FileKind& kind, const BinaryRow& partition,
+                                          int32_t bucket, const std::string& file_name,
+                                          const std::vector& data_file_names,
+                                          int64_t row_count) {
+        LinkedHashMap dv_ranges;
+        int32_t offset = 0;
+        for (const auto& data_file_name : data_file_names) {
+            dv_ranges.insert(data_file_name,
+                             DeletionVectorMeta(data_file_name, offset, /*length=*/10,
+                                                /*cardinality=*/std::nullopt));
+            offset += 10;
+        }
+        return IndexManifestEntry(
+            kind, partition, bucket,
+            std::make_shared(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX,
+                                            file_name, /*file_size=*/row_count * 10, row_count,
+                                            dv_ranges, /*external_path=*/std::nullopt));
+    }
+
     std::shared_ptr pool_;
     std::unique_ptr dir_;
 };
@@ -156,13 +175,55 @@ TEST_F(IndexManifestFileHandlerTest, BucketedCombinerUsesPartitionBucketAndIndex
     ASSERT_TRUE(found_bucket1);
 }
 
+TEST_F(IndexManifestFileHandlerTest, BucketedCombinerOverwritesDuplicateAddedEntries) {
+    ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/2));
+
+    auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get());
+    std::vector new_entries = {
+        MakeEntry(FileKind::Add(), partition, /*bucket=*/0,
+                  DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0-old", 10),
+        MakeEntry(FileKind::Add(), partition, /*bucket=*/0,
+                  DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0-new", 20)};
+
+    ASSERT_OK_AND_ASSIGN(std::string current_manifest,
+                         IndexManifestFileHandler::Write(
+                             /*previous_index_manifest=*/std::nullopt, new_entries,
+                             /*bucket_mode=*/2, index_manifest_file.get()));
+
+    std::vector written_entries;
+    ASSERT_OK(index_manifest_file->Read(current_manifest, /*filter=*/nullptr, &written_entries));
+    ASSERT_EQ(written_entries.size(), 1);
+    ASSERT_EQ(written_entries[0].index_file->FileName(), "dv-0-new");
+    ASSERT_EQ(written_entries[0].index_file->RowCount(), 20);
+}
+
+TEST_F(IndexManifestFileHandlerTest, GlobalCombinerOverwritesDuplicateAddedEntries) {
+    ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/4));
+
+    auto partition = BinaryRow::EmptyRow();
+    std::vector new_entries = {
+        MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0", 10),
+        MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0",
+                  20)};
+
+    ASSERT_OK_AND_ASSIGN(std::string current_manifest,
+                         IndexManifestFileHandler::Write(
+                             /*previous_index_manifest=*/std::nullopt, new_entries,
+                             /*bucket_mode=*/4, index_manifest_file.get()));
+
+    std::vector written_entries;
+    ASSERT_OK(index_manifest_file->Read(current_manifest, /*filter=*/nullptr, &written_entries));
+    ASSERT_EQ(written_entries.size(), 1);
+    ASSERT_EQ(written_entries[0].index_file->FileName(), "global-0");
+    ASSERT_EQ(written_entries[0].index_file->RowCount(), 20);
+}
+
 TEST_F(IndexManifestFileHandlerTest, DvWithBucketUnawareModeReturnsNotImplemented) {
     ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/-1));
 
     auto partition = BinaryRow::EmptyRow();
     std::vector new_entries = {
-        MakeEntry(FileKind::Add(), partition, /*bucket=*/0,
-                  DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0", 1)};
+        MakeDvEntry(FileKind::Add(), partition, /*bucket=*/0, "dv-0", {"data-0.orc"}, 1)};
 
     ASSERT_NOK_WITH_MSG(IndexManifestFileHandler::Write(
                             /*previous_index_manifest=*/std::nullopt, new_entries,
diff --git a/src/paimon/core/manifest/manifest_committable.h b/src/paimon/core/manifest/manifest_committable.h
index 8ba2b8bb..0fe72faf 100644
--- a/src/paimon/core/manifest/manifest_committable.h
+++ b/src/paimon/core/manifest/manifest_committable.h
@@ -42,15 +42,13 @@ class ManifestCommittable {
         : ManifestCommittable(identifier, std::nullopt) {}
 
     ManifestCommittable(int64_t identifier, std::optional watermark)
-        : ManifestCommittable(identifier, watermark, {}, {}, {}) {}
+        : ManifestCommittable(identifier, watermark, {}, {}) {}
 
     ManifestCommittable(int64_t identifier, std::optional watermark,
-                        const std::map& log_offsets,
                         const std::map& properties,
                         const std::vector>& commit_messages)
         : identifier_(identifier),
           watermark_(watermark),
-          log_offsets_(log_offsets),
           properties_(properties),
           commit_messages_(commit_messages) {}
 
@@ -62,10 +60,6 @@ class ManifestCommittable {
         return watermark_;
     }
 
-    const std::map& LogOffsets() const {
-        return log_offsets_;
-    }
-
     const std::map& Properties() const {
         return properties_;
     }
@@ -88,12 +82,6 @@ class ManifestCommittable {
         std::string watermark_str =
             watermark_ == std::nullopt ? "null" : std::to_string(watermark_.value());
 
-        std::vector log_offsets_str;
-        log_offsets_str.reserve(log_offsets_.size());
-        for (const auto& [key, value] : log_offsets_) {
-            log_offsets_str.emplace_back(fmt::format("{}: {}", key, value));
-        }
-
         std::vector properties_str;
         properties_str.reserve(properties_.size());
         for (const auto& [key, value] : properties_) {
@@ -101,16 +89,15 @@ class ManifestCommittable {
         }
 
         return fmt::format(
-            "ManifestCommittable {{identifier = {}, watermark = {}, logOffsets = {}, "
+            "ManifestCommittable {{identifier = {}, watermark = {}, "
             "commitMessages = {}, properties = {}}}",
-            identifier_, watermark_str, fmt::join(log_offsets_str, ", "),
-            fmt::join(commit_messages_str, ", "), fmt::join(properties_str, ", "));
+            identifier_, watermark_str, fmt::join(commit_messages_str, ", "),
+            fmt::join(properties_str, ", "));
     }
 
  private:
     int64_t identifier_;
     std::optional watermark_;
-    std::map log_offsets_;
     std::map properties_;
     std::vector> commit_messages_;
 };
diff --git a/src/paimon/core/manifest/manifest_committable_test.cpp b/src/paimon/core/manifest/manifest_committable_test.cpp
index 007f3366..00a8a30d 100644
--- a/src/paimon/core/manifest/manifest_committable_test.cpp
+++ b/src/paimon/core/manifest/manifest_committable_test.cpp
@@ -31,28 +31,6 @@ namespace paimon::test {
 
 class ManifestCommittableTest : public testing::Test {
  private:
-    bool IsEqualMap(const std::map& actual_map,
-                    const std::map& expected_map) {
-        if (expected_map.size() != actual_map.size()) {
-            return false;
-        }
-        for (const auto& kv : expected_map) {
-            const auto& key = kv.first;
-            const auto& value = kv.second;
-            auto iter = actual_map.find(key);
-            if (iter != actual_map.end()) {
-                if (iter->second == value) {
-                    continue;
-                } else {
-                    return false;
-                }
-            } else {
-                return false;
-            }
-        }
-        return true;
-    }
-
     std::vector> GetCommitMessages(const std::string& path,
                                                                   int32_t version) const {
         auto file_system = std::make_shared();
@@ -61,7 +39,7 @@ class ManifestCommittableTest : public testing::Test {
 
         EXPECT_OK_AND_ASSIGN(auto in_stream, file_system->Open(path));
         EXPECT_OK_AND_ASSIGN(
-            [[maybe_unused]] int32_t read_bytes,
+            [[maybe_unused]] int64_t read_bytes,
             in_stream->Read(reinterpret_cast(buffer.data()), buffer.size()));
         EXPECT_OK(in_stream->Close());
 
@@ -101,26 +79,22 @@ TEST_F(ManifestCommittableTest, TestSimple) {
         ASSERT_EQ(committable.Watermark().value(), 456);
     }
     {
-        std::map log_offsets = {{123, 444}, {234, 555}};
         std::map properties = {};
         std::vector> msgs =
             GetCommitMessages(paimon::test::GetDataDir() +
                                   "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
                               /*version=*/3);
-        ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, log_offsets,
-                                        properties, msgs);
-        ASSERT_TRUE(IsEqualMap(committable.LogOffsets(), log_offsets));
+        ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs);
+        ASSERT_EQ(committable.Properties(), properties);
         ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables()));
     }
     {
-        std::map log_offsets = {};
         std::map properties = {{"key1", "value1"}, {"key2", "value2"}};
         std::vector> msgs =
             GetCommitMessages(paimon::test::GetDataDir() +
                                   "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
                               /*version=*/3);
-        ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456,
-                                        /*log_offsets=*/{}, properties, msgs);
+        ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs);
         ASSERT_EQ(committable.Properties(), properties);
         ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables()));
     }
diff --git a/src/paimon/core/manifest/manifest_entry.cpp b/src/paimon/core/manifest/manifest_entry.cpp
index c19e32af..c0fbac0c 100644
--- a/src/paimon/core/manifest/manifest_entry.cpp
+++ b/src/paimon/core/manifest/manifest_entry.cpp
@@ -34,6 +34,22 @@ const std::shared_ptr& ManifestEntry::DataType() {
     return data_type;
 }
 
+int64_t ManifestEntry::RecordCount(const std::vector& entries) {
+    int64_t record_count = 0;
+    for (const auto& entry : entries) {
+        record_count += entry.File()->row_count;
+    }
+    return record_count;
+}
+
+std::optional ManifestEntry::NullableRecordCount(
+    const std::vector& entries) {
+    if (entries.empty()) {
+        return std::nullopt;
+    }
+    return RecordCount(entries);
+}
+
 int64_t ManifestEntry::RecordCountAdd(const std::vector& entries) {
     int64_t record_count = 0;
     for (const auto& entry : entries) {
diff --git a/src/paimon/core/manifest/manifest_entry.h b/src/paimon/core/manifest/manifest_entry.h
index 85e387ae..e39fa51c 100644
--- a/src/paimon/core/manifest/manifest_entry.h
+++ b/src/paimon/core/manifest/manifest_entry.h
@@ -39,6 +39,9 @@ namespace paimon {
 class ManifestEntry : public FileEntry {
  public:
     static const std::shared_ptr& DataType();
+    static int64_t RecordCount(const std::vector& manifest_entries);
+    static std::optional NullableRecordCount(
+        const std::vector& manifest_entries);
     static int64_t RecordCountAdd(const std::vector& manifest_entries);
     static int64_t RecordCountDelete(const std::vector& manifest_entries);
 
diff --git a/src/paimon/core/manifest/manifest_entry_serializer.cpp b/src/paimon/core/manifest/manifest_entry_serializer.cpp
index b8176d8f..053405b8 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer.cpp
+++ b/src/paimon/core/manifest/manifest_entry_serializer.cpp
@@ -54,7 +54,7 @@ Result ManifestEntrySerializer::ConvertFrom(int32_t version,
         return Status::Invalid("ManifestEntry convert from row failed, with null DataFileMeta");
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr meta,
-                           data_file_meta_serializer_.FromRow(*file))
+                           data_file_meta_serializer_.FromRow(*file));
     return ManifestEntry(file_kind, partition, bucket, total_buckets, meta);
 }
 
diff --git a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
index 3fa23466..2aa2db52 100644
--- a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
+++ b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp
@@ -34,12 +34,12 @@ namespace paimon::test {
 
 class ManifestEntrySerializerTest : public testing::Test {
  private:
-    std::shared_ptr GetDataFileMeta() {
+    std::shared_ptr GetDataFileMeta(int64_t row_count) {
         return std::make_shared(
-            "some_file_name", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
-            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
-            /*max_seq_no=*/32,
-            /*schema_id=*/1, /*level=*/2, /*extra_files=*/std::vector>(),
+            "some_file_name", 1024, row_count, DataFileMeta::EmptyMinKey(),
+            DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+            /*min_seq_no=*/16, /*max_seq_no=*/32, /*schema_id=*/1, /*level=*/2,
+            /*extra_files=*/std::vector>(),
             /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3,
             /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional(),
@@ -49,9 +49,9 @@ class ManifestEntrySerializerTest : public testing::Test {
 TEST_F(ManifestEntrySerializerTest, TestToFromRow) {
     auto pool = GetDefaultPool();
     std::vector entries = {
-        ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta()),
+        ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(8)),
         ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()), 1, 2,
-                      GetDataFileMeta())};
+                      GetDataFileMeta(8))};
     ManifestEntrySerializer serializer(pool);
     for (const auto& entry : entries) {
         ASSERT_OK_AND_ASSIGN(auto row, serializer.ToRow(entry));
@@ -60,4 +60,18 @@ TEST_F(ManifestEntrySerializerTest, TestToFromRow) {
         ASSERT_EQ(entry.ToString(), result_entry.ToString());
     }
 }
+
+TEST_F(ManifestEntrySerializerTest, TestNullableRecordCount) {
+    std::vector empty_entries;
+    ASSERT_FALSE(ManifestEntry::NullableRecordCount(empty_entries).has_value());
+
+    std::vector entries = {
+        ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(3)),
+        ManifestEntry(FileKind::Delete(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(8))};
+    ASSERT_EQ(11, ManifestEntry::RecordCount(entries));
+
+    std::optional record_count = ManifestEntry::NullableRecordCount(entries);
+    ASSERT_TRUE(record_count.has_value());
+    ASSERT_EQ(11, record_count.value());
+}
 }  // namespace paimon::test
diff --git a/src/paimon/core/manifest/manifest_entry_writer_factory.h b/src/paimon/core/manifest/manifest_entry_writer_factory.h
new file mode 100644
index 00000000..49620225
--- /dev/null
+++ b/src/paimon/core/manifest/manifest_entry_writer_factory.h
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/c/abi.h"
+#include "paimon/core/io/single_file_writer_factory.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry_writer.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/utils/path_factory.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class FileSystem;
+class ManifestEntryWriter;
+class MemoryPool;
+class PathFactory;
+class WriterBuilder;
+
+class ManifestEntryWriterFactory
+    : public SingleFileWriterFactory {
+ public:
+    ManifestEntryWriterFactory(const std::string& compression,
+                               std::function converter,
+                               const std::shared_ptr& pool,
+                               const std::shared_ptr& partition_type,
+                               const std::shared_ptr& file_system,
+                               const std::shared_ptr& path_factory,
+                               const std::shared_ptr& writer_builder)
+        : compression_(compression),
+          converter_(std::move(converter)),
+          pool_(pool),
+          partition_type_(partition_type),
+          file_system_(file_system),
+          path_factory_(path_factory),
+          writer_builder_(writer_builder) {}
+
+    Result>> CreateWriter()
+        const override {
+        auto writer =
+            std::make_unique(compression_, converter_, pool_, partition_type_);
+        PAIMON_RETURN_NOT_OK(writer->Init(file_system_, path_factory_->NewPath(), writer_builder_));
+        return std::unique_ptr>(
+            std::move(writer));
+    }
+
+ private:
+    std::string compression_;
+    std::function converter_;
+    std::shared_ptr pool_;
+    std::shared_ptr partition_type_;
+    std::shared_ptr file_system_;
+    std::shared_ptr path_factory_;
+    std::shared_ptr writer_builder_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index 52637bbb..22f2681f 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/manifest/manifest_file.h"
 
 #include 
+#include 
 #include 
 
 #include "arrow/c/abi.h"
@@ -27,7 +28,7 @@
 #include "paimon/core/io/rolling_file_writer.h"
 #include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/manifest/manifest_entry_serializer.h"
-#include "paimon/core/manifest/manifest_entry_writer.h"
+#include "paimon/core/manifest/manifest_entry_writer_factory.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/object_serializer.h"
@@ -56,7 +57,7 @@ ManifestFile::ManifestFile(const std::shared_ptr& file_system,
                            const std::shared_ptr& partition_type)
     : ObjectsFile(file_system, reader_builder, writer_builder,
                                  std::make_unique(pool), compression,
-                                 path_factory, pool),
+                                 path_factory, options.GetCache(), pool),
       target_file_size_(target_file_size),
       options_(options),
       partition_type_(partition_type) {}
@@ -102,16 +103,13 @@ Result> ManifestFile::Write(
         return Status::OK();
     };
 
-    auto create_file_writer = [&]() -> Result> {
-        auto writer = std::make_unique(options_.GetManifestCompression(),
-                                                            converter, pool_, partition_type_);
-        PAIMON_RETURN_NOT_OK(
-            writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder_));
-        return writer;
-    };
+    auto writer_factory = std::make_shared(
+        options_.GetManifestCompression(), converter, pool_, partition_type_,
+        options_.GetFileSystem(), path_factory_, writer_builder_);
     std::unique_ptr> writer =
         std::make_unique>(
-            target_file_size_, create_file_writer);
+            target_file_size_, /*target_file_row_num=*/std::numeric_limits::max(),
+            writer_factory);
     for (const auto& entry : entries) {
         auto s = writer->Write(entry);
         if (!s.ok()) {
diff --git a/src/paimon/core/manifest/manifest_file_meta_serializer.cpp b/src/paimon/core/manifest/manifest_file_meta_serializer.cpp
index 5018aeaa..c7042b27 100644
--- a/src/paimon/core/manifest/manifest_file_meta_serializer.cpp
+++ b/src/paimon/core/manifest/manifest_file_meta_serializer.cpp
@@ -72,14 +72,14 @@ Result ManifestFileMetaSerializer::ToRow(const ManifestFileMeta& reco
     if (!min_row_id) {
         writer.SetNullAt(11);
     } else {
-        writer.WriteInt(11, min_row_id.value());
+        writer.WriteLong(11, min_row_id.value());
     }
 
     auto max_row_id = record.MaxRowId();
     if (!max_row_id) {
         writer.SetNullAt(12);
     } else {
-        writer.WriteInt(12, max_row_id.value());
+        writer.WriteLong(12, max_row_id.value());
     }
     writer.Complete();
     return row;
diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp
index 38180003..8bb2c22e 100644
--- a/src/paimon/core/manifest/manifest_file_test.cpp
+++ b/src/paimon/core/manifest/manifest_file_test.cpp
@@ -43,9 +43,62 @@
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/testing/utils/binary_row_generator.h"
+#include "paimon/testing/utils/counting_cache_test_utils.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
+
+class CountingFileSystem : public FileSystem {
+ public:
+    Result> Open(const std::string& path) const override {
+        ++open_count;
+        return local_.Open(path);
+    }
+
+    Result> Create(const std::string& path,
+                                                 bool overwrite) const override {
+        return local_.Create(path, overwrite);
+    }
+
+    Status Mkdirs(const std::string& path) const override {
+        return local_.Mkdirs(path);
+    }
+
+    Status Rename(const std::string& src, const std::string& dst) const override {
+        return local_.Rename(src, dst);
+    }
+
+    Status Delete(const std::string& path, bool recursive = true) const override {
+        return local_.Delete(path, recursive);
+    }
+
+    Result> GetFileStatus(const std::string& path) const override {
+        ++get_file_status_count;
+        return local_.GetFileStatus(path);
+    }
+
+    Status ListDir(const std::string& directory,
+                   std::vector>* file_status_list) const override {
+        return local_.ListDir(directory, file_status_list);
+    }
+
+    Status ListFileStatus(
+        const std::string& path,
+        std::vector>* file_status_list) const override {
+        return local_.ListFileStatus(path, file_status_list);
+    }
+
+    Result Exists(const std::string& path) const override {
+        return local_.Exists(path);
+    }
+
+    mutable int open_count = 0;
+    mutable int get_file_status_count = 0;
+
+ private:
+    LocalFileSystem local_;
+};
+
 class ManifestFileTest : public testing::Test {
  public:
     std::vector ReadManifestEntry(const std::string& file_format_str,
@@ -183,6 +236,87 @@ TEST_F(ManifestFileTest, TestSimple) {
     ASSERT_EQ(expected_manifest_entries, manifest_entries);
 }
 
+TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) {
+    auto pool = GetDefaultPool();
+    auto counting_file_system = std::make_shared();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format,
+                         FileFormatFactory::Get("orc", {}));
+    std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+    auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr path_factory,
+        FileStorePathFactory::Create(root_path, unused_schema, /*partition_keys=*/{},
+                                     /*default_part_value=*/"", file_format->Identifier(),
+                                     /*data_file_prefix=*/"data-",
+                                     /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                                     /*global_index_external_path=*/std::nullopt,
+                                     /*index_file_in_data_file_dir=*/false, pool));
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr manifest_file,
+        ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory,
+                             /*target_file_size=*/1024, pool, options, unused_schema));
+
+    std::vector first_read;
+    ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1",
+                                  /*filter=*/nullptr, &first_read));
+    ASSERT_EQ(5, first_read.size());
+    ASSERT_EQ(1, counting_file_system->open_count);
+    ASSERT_EQ(0, counting_file_system->get_file_status_count);
+
+    std::vector filtered_read;
+    ASSERT_OK(manifest_file->Read(
+        "manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1",
+        [](const ManifestEntry& entry) -> Result { return entry.Kind() == FileKind::Add(); },
+        &filtered_read));
+    ASSERT_EQ(1, filtered_read.size());
+    ASSERT_EQ(2, counting_file_system->open_count);
+    ASSERT_EQ(0, counting_file_system->get_file_status_count);
+}
+
+TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) {
+    auto pool = GetDefaultPool();
+    auto counting_file_system = std::make_shared();
+    auto manifest_cache =
+        std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format,
+                         FileFormatFactory::Get("orc", {}));
+    std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+    auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())}));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr path_factory,
+        FileStorePathFactory::Create(root_path, unused_schema, /*partition_keys=*/{},
+                                     /*default_part_value=*/"", file_format->Identifier(),
+                                     /*data_file_prefix=*/"data-",
+                                     /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                                     /*global_index_external_path=*/std::nullopt,
+                                     /*index_file_in_data_file_dir=*/false, pool));
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}));
+    options.WithCache(manifest_cache);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr manifest_file,
+        ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory,
+                             /*target_file_size=*/1024, pool, options, unused_schema));
+
+    std::vector first_read;
+    ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1",
+                                  /*filter=*/nullptr, &first_read));
+    std::vector second_read;
+    ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1",
+                                  /*filter=*/nullptr, &second_read));
+
+    ASSERT_EQ(first_read, second_read);
+    ASSERT_EQ(1, counting_file_system->open_count);
+    ASSERT_EQ(0, counting_file_system->get_file_status_count);
+    ASSERT_EQ(2, manifest_cache->GetCount());
+    ASSERT_EQ(1, manifest_cache->SupplierCallCount());
+    ASSERT_EQ(1, manifest_cache->Size());
+}
+
 TEST_F(ManifestFileTest, TestWithNullCount) {
     auto pool = GetDefaultPool();
     auto manifest_entries =
diff --git a/src/paimon/core/manifest/manifest_list.cpp b/src/paimon/core/manifest/manifest_list.cpp
index 58b9cbdf..887b8845 100644
--- a/src/paimon/core/manifest/manifest_list.cpp
+++ b/src/paimon/core/manifest/manifest_list.cpp
@@ -43,15 +43,16 @@ ManifestList::ManifestList(const std::shared_ptr& file_system,
                            const std::shared_ptr& writer_builder,
                            const std::string& compression,
                            const std::shared_ptr& path_factory,
+                           const std::shared_ptr& cache,
                            const std::shared_ptr& pool)
     : ObjectsFile(file_system, reader_builder, writer_builder,
                                     std::make_unique(pool), compression,
-                                    std::move(path_factory), pool) {}
+                                    std::move(path_factory), cache, pool) {}
 
 Result> ManifestList::Create(
     const std::shared_ptr& fs, const std::shared_ptr& file_format,
     const std::string& compression, const std::shared_ptr& path_factory,
-    const std::shared_ptr& pool) {
+    const std::shared_ptr& cache, const std::shared_ptr& pool) {
     std::shared_ptr data_type =
         VersionedObjectSerializer::VersionType(ManifestFileMeta::DataType());
     // prepare format reader builder
@@ -71,7 +72,7 @@ Result> ManifestList::Create(
     std::shared_ptr manifest_list_path_factory =
         path_factory->CreateManifestListFactory();
     return std::unique_ptr(new ManifestList(
-        fs, reader_builder, writer_builder, compression, manifest_list_path_factory, pool));
+        fs, reader_builder, writer_builder, compression, manifest_list_path_factory, cache, pool));
 }
 
 Result> ManifestList::Write(
diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h
index 65b683bf..31959a5c 100644
--- a/src/paimon/core/manifest/manifest_list.h
+++ b/src/paimon/core/manifest/manifest_list.h
@@ -35,6 +35,7 @@
 
 namespace paimon {
 
+class Cache;
 class FileFormat;
 class FileSystem;
 class FileStorePathFactory;
@@ -52,7 +53,7 @@ class ManifestList : public ObjectsFile {
         const std::shared_ptr& file_system,
         const std::shared_ptr& file_format, const std::string& compression,
         const std::shared_ptr& path_factory,
-        const std::shared_ptr& pool);
+        const std::shared_ptr& cache, const std::shared_ptr& pool);
 
     /// Write several `ManifestFileMeta`s into a manifest list.
     ///
@@ -124,7 +125,7 @@ class ManifestList : public ObjectsFile {
                  const std::shared_ptr& reader_builder,
                  const std::shared_ptr& writer_builder,
                  const std::string& compression, const std::shared_ptr& path_factory,
-                 const std::shared_ptr& pool);
+                 const std::shared_ptr& cache, const std::shared_ptr& pool);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp
index 0f71afd9..889f18f2 100644
--- a/src/paimon/core/manifest/manifest_list_test.cpp
+++ b/src/paimon/core/manifest/manifest_list_test.cpp
@@ -23,6 +23,7 @@
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
+#include "paimon/core/core_options.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -51,8 +52,10 @@ class ManifestListTest : public testing::Test {
                                  /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
                                  /*global_index_external_path=*/std::nullopt,
                                  /*index_file_in_data_file_dir=*/false, pool));
-        EXPECT_OK_AND_ASSIGN(auto manifest_list, ManifestList::Create(file_system, file_format,
-                                                                      "zstd", path_factory, pool));
+        EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
+        EXPECT_OK_AND_ASSIGN(auto manifest_list,
+                             ManifestList::Create(file_system, file_format, "zstd", path_factory,
+                                                  options.GetCache(), pool));
         return manifest_list;
     }
 
diff --git a/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp b/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp
new file mode 100644
index 00000000..8982209a
--- /dev/null
+++ b/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/manifest/snapshot_live_manifest_entries.h"
+
+#include 
+#include 
+#include 
+
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/core/manifest/manifest_entry_serializer.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/memory/memory_segment.h"
+
+namespace paimon {
+namespace {
+
+constexpr int32_t kMagic = 0x534d4543;  // SMEC
+
+size_t NormalizeMaxSnapshots(int32_t max_snapshots) {
+    return static_cast(std::max(0, max_snapshots));
+}
+
+std::shared_ptr ToBytes(const MemorySegmentOutputStream& out,
+                               const std::shared_ptr& pool) {
+    auto bytes = Bytes::AllocateBytes(static_cast(out.CurrentSize()), pool.get());
+    int64_t offset = 0;
+    for (const auto& segment : out.Segments()) {
+        int64_t copy_size =
+            std::min(segment.Size(), static_cast(bytes->size()) - offset);
+        if (copy_size <= 0) {
+            break;
+        }
+        std::memcpy(bytes->data() + offset, segment.Data(), static_cast(copy_size));
+        offset += copy_size;
+    }
+    return bytes;
+}
+
+}  // namespace
+
+SnapshotLiveManifestEntries::SnapshotLiveManifestEntries(int32_t max_snapshots)
+    : max_snapshots_(max_snapshots) {}
+
+std::optional SnapshotLiveManifestEntries::LatestBeforeOrEqual(
+    int64_t snapshot_id) const {
+    auto iter = entries_by_snapshot_.upper_bound(snapshot_id);
+    if (iter == entries_by_snapshot_.begin()) {
+        return std::optional();
+    }
+    --iter;
+    return Entry{iter->first, iter->second};
+}
+
+void SnapshotLiveManifestEntries::Put(int64_t snapshot_id, std::vector&& entries) {
+    if (NormalizeMaxSnapshots(max_snapshots_) == 0) {
+        return;
+    }
+    entries_by_snapshot_[snapshot_id] =
+        std::make_shared>(std::move(entries));
+    EvictIfNeeded();
+}
+
+size_t SnapshotLiveManifestEntries::Size() const {
+    return entries_by_snapshot_.size();
+}
+
+Result> SnapshotLiveManifestEntries::Serialize(
+    const std::shared_ptr& pool) const {
+    MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool);
+    out.WriteValue(kMagic);
+    out.WriteValue(static_cast(entries_by_snapshot_.size()));
+
+    ManifestEntrySerializer serializer(pool);
+    for (const auto& [snapshot_id, entries] : entries_by_snapshot_) {
+        out.WriteValue(snapshot_id);
+        PAIMON_RETURN_NOT_OK(serializer.SerializeList(*entries, &out));
+    }
+    return ToBytes(out, pool);
+}
+
+Result SnapshotLiveManifestEntries::Deserialize(
+    const MemorySegment& segment, int32_t max_snapshots, const std::shared_ptr& pool) {
+    SnapshotLiveManifestEntries snapshot_live_manifest_entries(max_snapshots);
+    if (segment.Data() == nullptr || segment.Size() == 0) {
+        return snapshot_live_manifest_entries;
+    }
+
+    auto bytes = segment.GetOrCreateHeapMemory(pool.get());
+    auto input_stream = std::make_shared(bytes->data(), bytes->size());
+    DataInputStream in(input_stream);
+
+    PAIMON_ASSIGN_OR_RAISE(int32_t magic, in.ReadValue());
+    if (magic != kMagic) {
+        return Status::Invalid("invalid snapshot live manifest entries magic");
+    }
+    PAIMON_ASSIGN_OR_RAISE(int32_t snapshot_count, in.ReadValue());
+    if (snapshot_count < 0) {
+        return Status::Invalid("snapshot live manifest entries snapshot count is negative");
+    }
+
+    ManifestEntrySerializer serializer(pool);
+    for (int32_t i = 0; i < snapshot_count; i++) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, in.ReadValue());
+        PAIMON_ASSIGN_OR_RAISE(std::vector entries, serializer.DeserializeList(&in));
+        snapshot_live_manifest_entries.entries_by_snapshot_[snapshot_id] =
+            std::make_shared>(std::move(entries));
+    }
+    snapshot_live_manifest_entries.EvictIfNeeded();
+    return snapshot_live_manifest_entries;
+}
+
+void SnapshotLiveManifestEntries::EvictIfNeeded() {
+    size_t max_snapshots = NormalizeMaxSnapshots(max_snapshots_);
+    while (entries_by_snapshot_.size() > max_snapshots) {
+        entries_by_snapshot_.erase(entries_by_snapshot_.begin());
+    }
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/manifest/snapshot_live_manifest_entries.h b/src/paimon/core/manifest/snapshot_live_manifest_entries.h
new file mode 100644
index 00000000..acf1d8e7
--- /dev/null
+++ b/src/paimon/core/manifest/snapshot_live_manifest_entries.h
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class Bytes;
+class MemoryPool;
+class MemorySegment;
+
+/// Live manifest entries retained for one bucket across multiple snapshots.
+///
+/// This value object owns merged live manifest entries by snapshot id. It does not own or access a
+/// cache; callers are responsible for storing the serialized bytes in the cache layer.
+class SnapshotLiveManifestEntries {
+ public:
+    struct Entry {
+        int64_t snapshot_id;
+        std::shared_ptr> entries;
+    };
+
+    explicit SnapshotLiveManifestEntries(int32_t max_snapshots);
+
+    std::optional LatestBeforeOrEqual(int64_t snapshot_id) const;
+    void Put(int64_t snapshot_id, std::vector&& entries);
+    size_t Size() const;
+
+    Result> Serialize(const std::shared_ptr& pool) const;
+    static Result Deserialize(const MemorySegment& segment,
+                                                           int32_t max_snapshots,
+                                                           const std::shared_ptr& pool);
+
+ private:
+    void EvictIfNeeded();
+
+    std::map>> entries_by_snapshot_;
+    int32_t max_snapshots_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp
index 45b5f16e..aae80509 100644
--- a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp
@@ -64,7 +64,11 @@ Status AggregateMergeFunction::Add(KeyValue&& kv) {
     // mark the current row for deletion and initialize the row with input values.
     if (remove_record_on_delete_ && kv.value_kind == RowKind::Delete()) {
         current_delete_row_ = true;
-        row_ = std::make_unique(getters_.size());
+        if (row_) {
+            row_->ResetFields();
+        } else {
+            row_ = std::make_unique(getters_.size());
+        }
         for (size_t i = 0; i < getters_.size(); i++) {
             row_->SetField(i, getters_[i](*(kv.value)));
         }
diff --git a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h
index 243ce6f7..7a1a07c4 100644
--- a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h
+++ b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h
@@ -54,7 +54,11 @@ class AggregateMergeFunction : public MergeFunction {
     void Reset() override {
         latest_kv_ = std::nullopt;
         current_delete_row_ = false;
-        row_ = std::make_unique(getters_.size());
+        if (row_) {
+            row_->ResetFields();
+        } else {
+            row_ = std::make_unique(getters_.size());
+        }
         for (const auto& agg : aggregators_) {
             agg->Reset();
         }
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
index beb1aeb9..0901bcad 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
@@ -120,6 +120,14 @@ TEST_F(FieldListaggAggTest, TestDistinctNoDuplicates) {
     ASSERT_EQ(DataDefine::GetVariantValue(ret), "a b c d");
 }
 
+TEST_F(FieldListaggAggTest, TestDistinctWithEmptyDelimiterFallsBackToWhitespace) {
+    ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg("", true));
+
+    // Empty delimiter falls back to whitespace, so the repeated "b" is removed.
+    auto ret = agg->Agg(std::string_view("a b"), std::string_view("b c"));
+    ASSERT_EQ(DataDefine::GetVariantValue(ret), "a b c");
+}
+
 TEST_F(FieldListaggAggTest, TestDistinctEmptyInput) {
     ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(";", true));
 
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
index e4c04dcf..955f23ed 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
+++ b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
@@ -21,6 +21,7 @@
 #include 
 #include 
 
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h"
 
 namespace paimon {
@@ -60,16 +61,32 @@ class FieldMaxAgg : public FieldAggregator {
             case arrow::Type::type::INT32:
             case arrow::Type::type::DATE32:
             case arrow::Type::type::INT64:
-            case arrow::Type::type::FLOAT:
-            case arrow::Type::type::DOUBLE:
             case arrow::Type::type::TIMESTAMP:
-            case arrow::Type::type::DECIMAL:
+            case arrow::Type::type::DECIMAL128:
             case arrow::Type::type::STRING:
             case arrow::Type::type::BINARY:
                 return FieldMaxFunc([](const VariantType& accumulator,
                                        const VariantType& input_field) -> VariantType {
                     return accumulator < input_field ? input_field : accumulator;
                 });
+            case arrow::Type::type::FLOAT:
+                return FieldMaxFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? input_field : accumulator;
+                });
+            case arrow::Type::type::DOUBLE:
+                return FieldMaxFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? input_field : accumulator;
+                });
             default:
                 return Status::Invalid(
                     fmt::format("type {} not support in FieldMaxAgg", field_type->ToString()));
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
index 69e10736..96c25b48 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
+++ b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
@@ -21,6 +21,7 @@
 #include 
 #include 
 
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h"
 
 namespace paimon {
@@ -60,16 +61,32 @@ class FieldMinAgg : public FieldAggregator {
             case arrow::Type::type::INT32:
             case arrow::Type::type::DATE32:
             case arrow::Type::type::INT64:
-            case arrow::Type::type::FLOAT:
-            case arrow::Type::type::DOUBLE:
             case arrow::Type::type::TIMESTAMP:
-            case arrow::Type::type::DECIMAL:
+            case arrow::Type::type::DECIMAL128:
             case arrow::Type::type::STRING:
             case arrow::Type::type::BINARY:
                 return FieldMinFunc([](const VariantType& accumulator,
                                        const VariantType& input_field) -> VariantType {
                     return accumulator < input_field ? accumulator : input_field;
                 });
+            case arrow::Type::type::FLOAT:
+                return FieldMinFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? accumulator : input_field;
+                });
+            case arrow::Type::type::DOUBLE:
+                return FieldMinFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? accumulator : input_field;
+                });
             default:
                 return Status::Invalid(
                     fmt::format("type {} not support in FieldMinAgg", field_type->ToString()));
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
index 0f79a527..c51b76cd 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
+#include 
+#include 
+#include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -40,6 +44,41 @@ class DataType;
 
 namespace paimon::test {
 
+template 
+void AssertSameFloatingPoint(T actual, T expected) {
+    if (std::isnan(expected)) {
+        ASSERT_TRUE(std::isnan(actual));
+        return;
+    }
+    ASSERT_EQ(actual, expected);
+    if (expected == static_cast(0.0)) {
+        ASSERT_EQ(std::signbit(actual), std::signbit(expected));
+    }
+}
+
+template 
+void CheckJavaCompatibleFloatingPointMinMax(const std::shared_ptr& type) {
+    const T infinity = std::numeric_limits::infinity();
+    const T nan = std::numeric_limits::quiet_NaN();
+    // This is the total order defined by Java Float.compare and Double.compare.
+    const std::array values = {-infinity, -static_cast(0.0), static_cast(0.0), infinity,
+                                     nan};
+
+    ASSERT_OK_AND_ASSIGN(auto field_min_agg, FieldMinAgg::Create(type));
+    ASSERT_OK_AND_ASSIGN(auto field_max_agg, FieldMaxAgg::Create(type));
+    for (size_t i = 0; i < values.size(); ++i) {
+        for (size_t j = 0; j < values.size(); ++j) {
+            VariantType min_result = field_min_agg->Agg(values[i], values[j]);
+            AssertSameFloatingPoint(DataDefine::GetVariantValue(min_result),
+                                    values[std::min(i, j)]);
+
+            VariantType max_result = field_max_agg->Agg(values[i], values[j]);
+            AssertSameFloatingPoint(DataDefine::GetVariantValue(max_result),
+                                    values[std::max(i, j)]);
+        }
+    }
+}
+
 TEST(FieldMinMaxAggTest, TestSimple) {
     {
         ASSERT_OK_AND_ASSIGN(auto field_min_agg, FieldMinAgg::Create(arrow::int32()));
@@ -53,6 +92,11 @@ TEST(FieldMinMaxAggTest, TestSimple) {
     }
 }
 
+TEST(FieldMinMaxAggTest, TestJavaCompatibleFloatingPointOrder) {
+    CheckJavaCompatibleFloatingPointMinMax(arrow::float32());
+    CheckJavaCompatibleFloatingPointMinMax(arrow::float64());
+}
+
 TEST(FieldMinMaxAggTest, TestInvalidType) {
     auto field_min_agg = FieldMinAgg::Create(arrow::boolean());
     ASSERT_FALSE(field_min_agg.ok());
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp b/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp
index 021f0202..a543e896 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp
@@ -73,7 +73,7 @@ Result FieldSumAgg::CreateSumFunc(
                                  DataDefine::GetVariantValue(input_field);
                     return sum;
                 });
-        case arrow::Type::type::DECIMAL: {
+        case arrow::Type::type::DECIMAL128: {
             return FieldSumFunc(
                 [](const VariantType& accumulator, const VariantType& input_field) -> VariantType {
                     auto v1 = DataDefine::GetVariantValue(accumulator);
@@ -122,7 +122,7 @@ Result FieldSumAgg::CreateNegFunc(
                 auto value = DataDefine::GetVariantValue(input_field);
                 return (-value);
             });
-        case arrow::Type::type::DECIMAL: {
+        case arrow::Type::type::DECIMAL128: {
             return FieldNegFunc([](const VariantType& input_field) -> VariantType {
                 auto value = DataDefine::GetVariantValue(input_field);
                 return Decimal(value.Precision(), value.Scale(), -value.Value());
diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
index ecd2af7e..3efd40f6 100644
--- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
@@ -80,9 +80,10 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog(
                            GenerateKeyValueConsumer());
     std::vector> reader_holders;
 
+    auto before = ExtractFilesFromSections(sections);
     std::unique_ptr compact_file_writer;
     if (rewrite_compact_file) {
-        compact_file_writer = CreateRollingRowWriter(output_level);
+        PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level));
     }
     // TODO(xinyu.lxy): produce changelog
     ScopeGuard write_guard([&]() -> void {
@@ -102,7 +103,6 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog(
     if (compact_file_writer) {
         PAIMON_RETURN_NOT_OK(compact_file_writer->Close());
     }
-    auto before = ExtractFilesFromSections(sections);
     std::vector> after;
     if (compact_file_writer) {
         PAIMON_ASSIGN_OR_RAISE(after, compact_file_writer->GetResult());
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
index c8c0b182..994b0e3f 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
@@ -18,6 +18,8 @@
 
 #include "paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h"
 
+#include 
+
 #include "paimon/common/table/special_fields.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h"
 #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h"
@@ -90,6 +92,7 @@ LookupMergeTreeCompactRewriter::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor()));
+
     return std::unique_ptr(new LookupMergeTreeCompactRewriter(
         std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(),
         trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache,
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
index 4aaf704f..06c5f47f 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
@@ -17,6 +17,7 @@
  */
 
 #pragma once
+
 #include "arrow/api.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -26,7 +27,9 @@
 #include "paimon/core/mergetree/lookup/remote_lookup_file_manager.h"
 #include "paimon/core/mergetree/lookup_levels.h"
 #include "paimon/core/schema/table_schema.h"
+
 namespace paimon {
+
 /// A `MergeTreeCompactRewriter` which produces changelog files by lookup for the compaction
 /// involving level 0 files.
 template 
@@ -87,7 +90,6 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter {
     Result>> NotifyRewriteCompactAfter(
         const std::vector>& files) override;
 
- private:
     std::unique_ptr> lookup_levels_;
     std::shared_ptr dv_maintainer_;
     std::shared_ptr remote_lookup_file_manager_;
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
index d5836d09..c734d5e9 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
@@ -55,7 +55,9 @@
 #include "paimon/testing/utils/io_exception_helper.h"
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
+
 namespace paimon::test {
+
 class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam {
  public:
     void SetUp() override {
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
index a8f56781..19b8bc36 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
@@ -21,6 +21,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -30,12 +31,17 @@
 #include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/compact/compact_deletion_file.h"
+#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/level_sorted_run.h"
 #include "paimon/core/mergetree/levels.h"
 #include "paimon/core/mergetree/sorted_run.h"
 #include "paimon/core/stats/simple_stats.h"
+#include "paimon/fs/file_system_factory.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -50,6 +56,10 @@ class InlineExecutor final : public Executor {
     }
 
     void ShutdownNow() override {}
+
+    uint32_t GetThreadNum() const override {
+        return 1;
+    }
 };
 
 class QueuedExecutor final : public Executor {
@@ -60,6 +70,10 @@ class QueuedExecutor final : public Executor {
 
     void ShutdownNow() override {}
 
+    uint32_t GetThreadNum() const override {
+        return 1;
+    }
+
     void RunAll() {
         for (auto& task : tasks_) {
             task();
@@ -403,6 +417,80 @@ TEST_F(MergeTreeCompactManagerTest, TestTriggerFullCompaction) {
     }
 }
 
+TEST_F(MergeTreeCompactManagerTest, TestFullCompactionRewritesEachMaxLevelFile) {
+    std::vector> files = {
+        ToFile(LevelMinMax(2, 1, 3), /*max_sequence=*/3),
+        ToFile(LevelMinMax(2, 4, 6), /*max_sequence=*/6)};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr levels, CreateLevels(files));
+
+    auto manager = std::make_shared(
+        levels, std::make_shared(TestStrategy()), comparator_,
+        /*compaction_file_size=*/2,
+        /*num_sorted_run_stop_trigger=*/std::numeric_limits::max(),
+        std::make_shared(/*expected_drop_delete=*/true),
+        /*metrics_reporter=*/nullptr,
+        /*dv_maintainer=*/nullptr,
+        /*lazy_gen_deletion_file=*/false,
+        /*need_lookup=*/false,
+        /*force_rewrite_all_files=*/true,
+        /*force_keep_delete=*/false, std::make_shared(),
+        std::make_shared());
+
+    ASSERT_OK(manager->TriggerCompaction(/*full_compaction=*/true));
+    ASSERT_OK_AND_ASSIGN(std::optional> compact_result,
+                         manager->GetCompactionResult(/*blocking=*/true));
+    ASSERT_TRUE(compact_result.has_value());
+    ASSERT_EQ(compact_result.value()->Before(), files);
+    ASSERT_EQ(compact_result.value()->After().size(), 2);
+    for (const auto& file : compact_result.value()->After()) {
+        ASSERT_EQ(file->file_name.rfind("rewrite-", /*pos=*/0), 0);
+    }
+}
+
+TEST_F(MergeTreeCompactManagerTest, TestCompactionGeneratesDeletionFileEagerly) {
+    std::vector> files = {
+        ToFile(LevelMinMax(0, 1, 3), /*max_sequence=*/0),
+        ToFile(LevelMinMax(1, 1, 5), /*max_sequence=*/1)};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr levels, CreateLevels(files));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr fs,
+                         FileSystemFactory::Get("local", dir->Str(), {}));
+    auto path_factory = std::make_shared(dir->Str());
+    auto dv_index_file =
+        std::make_shared(fs, path_factory, /*bitmap64=*/false, pool_);
+    auto dv_maintainer = std::make_shared(
+        dv_index_file, std::map>{});
+    ASSERT_OK(dv_maintainer->NotifyNewDeletion("remaining-file", /*position=*/0));
+
+    auto manager = std::make_shared(
+        levels, std::make_shared(TestStrategy()), comparator_,
+        /*compaction_file_size=*/2,
+        /*num_sorted_run_stop_trigger=*/std::numeric_limits::max(),
+        std::make_shared(/*expected_drop_delete=*/true),
+        /*metrics_reporter=*/nullptr, dv_maintainer,
+        /*lazy_gen_deletion_file=*/false,
+        /*need_lookup=*/false,
+        /*force_rewrite_all_files=*/false,
+        /*force_keep_delete=*/false, std::make_shared(),
+        std::make_shared());
+
+    ASSERT_OK(manager->TriggerCompaction(/*full_compaction=*/false));
+    ASSERT_OK_AND_ASSIGN(std::optional> compact_result,
+                         manager->GetCompactionResult(/*blocking=*/true));
+    ASSERT_TRUE(compact_result.has_value());
+    std::shared_ptr deletion_file = compact_result.value()->DeletionFile();
+    ASSERT_NE(deletion_file, nullptr);
+    ASSERT_NE(std::dynamic_pointer_cast(deletion_file), nullptr);
+
+    ASSERT_OK_AND_ASSIGN(std::optional> index_file,
+                         deletion_file->GetOrCompute());
+    ASSERT_TRUE(index_file.has_value());
+    ASSERT_EQ(index_file.value()->IndexType(), DeletionVectorsIndexFile::DELETION_VECTORS_INDEX);
+    ASSERT_OK_AND_ASSIGN(bool exists, dv_index_file->Exists(index_file.value()));
+    ASSERT_TRUE(exists);
+}
+
 TEST_F(MergeTreeCompactManagerTest, TestRejectReentrantFullCompaction) {
     std::vector inputs = {LevelMinMax(0, 1, 3), LevelMinMax(1, 2, 5),
                                        LevelMinMax(1, 6, 7)};
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
index 519c1701..1b64be2d 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
@@ -18,20 +18,22 @@
 #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h"
 
 #include 
+#include 
+#include 
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/scope_guard.h"
-#include "paimon/core/io/key_value_data_file_writer.h"
+#include "paimon/core/io/key_value_data_file_writer_factory.h"
 #include "paimon/core/io/key_value_meta_projection_consumer.h"
 #include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/io/row_to_arrow_array_converter.h"
-#include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/format/file_format.h"
-#include "paimon/format/writer_builder.h"
 #include "paimon/read_context.h"
 namespace paimon {
 MergeTreeCompactRewriter::MergeTreeCompactRewriter(
@@ -100,6 +102,7 @@ Result> MergeTreeCompactRewriter::Crea
         [](int32_t output_level) -> Result>> {
         return std::shared_ptr>();
     };
+
     return std::unique_ptr(new MergeTreeCompactRewriter(
         partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema,
         write_schema, std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read),
@@ -130,38 +133,28 @@ std::vector> MergeTreeCompactRewriter::ExtractFile
     return files;
 }
 
-std::unique_ptr
+Result>
 MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) {
-    auto create_file_writer = [this, level]()
-        -> Result>>> {
-        ::ArrowSchema arrow_schema{};
-        ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); });
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema));
-        auto format = options_.GetWriteFileFormat(level);
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr writer_builder,
-            format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize()));
-        writer_builder->WithMemoryPool(pool_);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema));
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor,
-                               format->CreateStatsExtractor(&arrow_schema));
-        auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status {
-            ArrowArrayMove(key_value_batch.batch.get(), array);
-            return Status::OK();
-        };
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
-                               CreateDataFilePathFactory(format->Identifier()));
-
-        auto writer = std::make_unique(
-            options_.GetWriteFileCompression(level), converter, schema_id_, level,
-            FileSource::Compact(), trimmed_primary_keys_, stats_extractor, write_schema_,
-            data_file_path_factory->IsExternalPath(), pool_);
-        PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(),
-                                          data_file_path_factory->NewPath(), writer_builder));
-        return writer;
-    };
+    auto format = options_.GetWriteFileFormat(level);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
+                           CreateDataFilePathFactory(format->Identifier()));
+    std::shared_ptr>> factory;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, level, FileSource::Compact(),
+            trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true,
+            plan_factory, pool_);
+    } else {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, level, FileSource::Compact(),
+            trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_);
+    }
     return std::make_unique(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer);
+        options_.GetTargetFileSize(/*has_primary_key=*/true),
+        /*target_file_row_num=*/std::numeric_limits::max(), factory);
 }
 
 Result
@@ -259,9 +252,10 @@ Result MergeTreeCompactRewriter::RewriteCompaction(
     int32_t output_level, bool drop_delete, const std::vector>& sections) {
     PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer,
                            GenerateKeyValueConsumer());
-
+    auto before = ExtractFilesFromSections(sections);
     std::vector> reader_holders;
-    auto rolling_writer = CreateRollingRowWriter(output_level);
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr rolling_writer,
+                           CreateRollingRowWriter(output_level));
 
     ScopeGuard write_guard([&]() -> void {
         rolling_writer->Abort();
@@ -278,7 +272,6 @@ Result MergeTreeCompactRewriter::RewriteCompaction(
 
     PAIMON_RETURN_NOT_OK(rolling_writer->Close());
 
-    auto before = ExtractFilesFromSections(sections);
     NotifyRewriteCompactBefore(before);
     PAIMON_ASSIGN_OR_RAISE(std::vector> after,
                            rolling_writer->GetResult());
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
index 3c7ab00b..c9c62470 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
@@ -32,6 +32,7 @@
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/file_store_path_factory_cache.h"
 namespace paimon {
+
 /// Default `CompactRewriter` for merge trees.
 class MergeTreeCompactRewriter : public CompactRewriter {
  public:
@@ -89,7 +90,7 @@ class MergeTreeCompactRewriter : public CompactRewriter {
     using KeyValueConsumerCreator =
         AsyncKeyValueProducerAndConsumer::ConsumerCreator;
 
-    std::unique_ptr CreateRollingRowWriter(int32_t level);
+    Result> CreateRollingRowWriter(int32_t level);
 
     Result GenerateKeyValueConsumer() const;
 
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
index d546035c..cb26540a 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
@@ -112,6 +112,14 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) {
     // load table schema
     SchemaManager schema_manager(fs, table_path);
     ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
+    auto options = table_schema->Options();
+    options[Options::TARGET_FILE_ROW_NUM] = "1";
+    auto logical_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr configured_schema,
+        TableSchema::Create(table_schema->Id(), logical_schema, table_schema->PartitionKeys(),
+                            table_schema->PrimaryKeys(), options));
+    table_schema = std::shared_ptr(std::move(configured_schema));
     ASSERT_OK_AND_ASSIGN(
         auto rewriter,
         CreateCompactRewriter(table_path, table_schema, /*bucket=*/1,
@@ -119,11 +127,12 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) {
 
     // generate sorted runs and rewrite
     ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1,
-                                                       /*partition=*/{{"f1", "10"}}))
+                                                       /*partition=*/{{"f1", "10"}}));
     ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite(
                                                   /*output_level=*/5, /*drop_delete=*/true, runs));
     // check compact result
     ASSERT_EQ(4, compact_result.Before().size());
+    // Compaction must not honor target-file-row-num; all seven rows stay in one output file.
     ASSERT_EQ(1, compact_result.After().size());
     const auto& compact_file_meta = compact_result.After()[0];
     auto expected_file_meta = std::make_shared(
@@ -216,7 +225,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestNotDropDelete) {
 
     // generate sorted runs and rewrite
     ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1,
-                                                       /*partition=*/{{"f1", "10"}}))
+                                                       /*partition=*/{{"f1", "10"}}));
     ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite(
                                                   /*output_level=*/5, /*drop_delete=*/false, runs));
     // check compact result
@@ -287,7 +296,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestIOException) {
 
         // generate sorted runs and rewrite
         ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1,
-                                                           /*partition=*/{{"f1", "10"}}))
+                                                           /*partition=*/{{"f1", "10"}}));
         // rewrite may trigger I/O exception
         ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
         io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);
diff --git a/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp b/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp
index 22c1fb1a..85ed9e27 100644
--- a/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp
+++ b/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp
@@ -271,7 +271,11 @@ void PartialUpdateMergeFunction::Reset() {
     current_key_.reset();
     meet_insert_ = false;
     not_null_column_filled_ = false;
-    row_ = std::make_unique(getters_.size());
+    if (row_) {
+        row_->ResetFields();
+    } else {
+        row_ = std::make_unique(getters_.size());
+    }
     last_seq_num_ = 0;
     for (auto& [_, agg] : field_aggregators_) {
         assert(agg);
@@ -306,7 +310,11 @@ Status PartialUpdateMergeFunction::Add(KeyValue&& moved_kv) {
         if (remove_record_on_delete_) {
             if (kv.value_kind == RowKind::Delete()) {
                 current_delete_row_ = true;
-                row_ = std::make_unique(getters_.size());
+                if (row_) {
+                    row_->ResetFields();
+                } else {
+                    row_ = std::make_unique(getters_.size());
+                }
                 InitRowAndHoldData(std::move(kv.value));
             } else if (!not_null_column_filled_) {
                 InitRowAndHoldData(std::move(kv.value));
diff --git a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp
index 66c6dce4..cf1174f8 100644
--- a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp
+++ b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp
@@ -142,7 +142,7 @@ Result InMemorySortBuffer::EstimateMemoryUse(const std::shared_ptrlength() * sizeof(double);
         case arrow::Type::type::TIMESTAMP:
             return null_bits_size_in_bytes + array->length() * sizeof(int64_t);
-        case arrow::Type::type::DECIMAL:
+        case arrow::Type::type::DECIMAL128:
             return null_bits_size_in_bytes + array->length() * sizeof(Decimal::int128_t);
         case arrow::Type::type::STRING:
         case arrow::Type::type::BINARY: {
diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp
index 52f4dee4..6d962572 100644
--- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp
+++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp
@@ -19,8 +19,10 @@
 
 #include "fmt/format.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/mergetree/lookup/file_position.h"
 #include "paimon/core/mergetree/lookup/positioned_key_value.h"
+
 namespace paimon {
 
 RemoteLookupFileManager::RemoteLookupFileManager(
@@ -53,7 +55,7 @@ Result> RemoteLookupFileManager::GenRemoteLookupFi
     // Get the file size from the local file system
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr local_file_status,
                            file_system_->GetFileStatus(local_file_path));
-    auto length = static_cast(local_file_status->GetLen());
+    int64_t length = local_file_status->GetLen();
 
     std::string remote_sst_name = lookup_levels->NewRemoteSst(file, length);
     std::string remote_sst_path = RemoteSstPath(file, remote_sst_name);
@@ -105,32 +107,46 @@ Status RemoteLookupFileManager::CopyRemoteToLocal(const std::string& remote_path
 Status RemoteLookupFileManager::CopyFromInputToOutput(
     std::unique_ptr&& input_stream,
     std::unique_ptr&& output_stream) const {
+    ScopeGuard input_close_guard([&input_stream]() -> void {
+        Status s = input_stream->Close();
+        (void)s;
+    });
+    ScopeGuard output_close_guard([&output_stream]() -> void {
+        Status s = output_stream->Close();
+        (void)s;
+    });
+
     auto buffer = std::make_shared(kBufferSize, pool_.get());
-    PAIMON_ASSIGN_OR_RAISE(uint64_t total_length, input_stream->Length());
-    uint64_t write_size = 0;
+    PAIMON_ASSIGN_OR_RAISE(int64_t total_length, input_stream->Length());
+    PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(total_length, "input stream length"));
+    int64_t write_size = 0;
     while (write_size < total_length) {
-        uint64_t current_read_size = std::min(total_length - write_size, kBufferSize);
-        PAIMON_ASSIGN_OR_RAISE(int32_t bytes_read,
+        int64_t current_read_size =
+            std::min(total_length - write_size, static_cast(kBufferSize));
+        PAIMON_ASSIGN_OR_RAISE(int64_t bytes_read,
                                input_stream->Read(buffer->data(), current_read_size));
-        if (static_cast(bytes_read) != current_read_size) {
-            return Status::Invalid(
-                fmt::format("CopyFromInputToOutput failed: expected read {} bytes, while "
-                            "actual read {} bytes",
-                            current_read_size, bytes_read));
+        if (bytes_read != current_read_size) {
+            return Status::Invalid(fmt::format(
+                "CopyFromInputToOutput failed: expected read {} bytes, while actual read {} bytes",
+                current_read_size, bytes_read));
         }
-        PAIMON_ASSIGN_OR_RAISE(int32_t bytes_written,
+        PAIMON_ASSIGN_OR_RAISE(int64_t bytes_written,
                                output_stream->Write(buffer->data(), bytes_read));
         if (bytes_written != bytes_read) {
             return Status::Invalid(
-                fmt::format("CopyFromInputToOutput failed: expected write {} bytes, while "
-                            "actual write {} bytes",
+                fmt::format("CopyFromInputToOutput failed: expected write {} bytes, while actual "
+                            "write {} bytes",
                             bytes_read, bytes_written));
         }
         write_size += current_read_size;
     }
     PAIMON_RETURN_NOT_OK(output_stream->Flush());
-    PAIMON_RETURN_NOT_OK(output_stream->Close());
-    PAIMON_RETURN_NOT_OK(input_stream->Close());
+    Status output_close_status = output_stream->Close();
+    output_close_guard.Release();
+    PAIMON_RETURN_NOT_OK(output_close_status);
+    Status input_close_status = input_stream->Close();
+    input_close_guard.Release();
+    PAIMON_RETURN_NOT_OK(input_close_status);
     return Status::OK();
 }
 template Result>
diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h
index 1d0b50a8..d8f9e7da 100644
--- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h
+++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h
@@ -52,7 +52,6 @@ class RemoteLookupFileManager {
 
     static constexpr uint64_t kBufferSize = 1024 * 1024;
 
- private:
     int32_t level_threshold_;
     std::shared_ptr pool_;
     std::shared_ptr path_factory_;
diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
index 54ed805d..cd8f3e18 100644
--- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
+++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
@@ -277,9 +277,9 @@ TEST_F(RemoteLookupFileManagerTest, TryToDownloadLargeFileAcrossMultipleBuffers)
         for (uint64_t i = 0; i < file_size; ++i) {
             write_buffer[i] = static_cast(i % 251);
         }
-        ASSERT_OK_AND_ASSIGN(int32_t bytes_written,
+        ASSERT_OK_AND_ASSIGN(int64_t bytes_written,
                              output_stream->Write(write_buffer.data(), file_size));
-        ASSERT_EQ(static_cast(bytes_written), file_size);
+        ASSERT_EQ(bytes_written, file_size);
         ASSERT_OK(output_stream->Flush());
         ASSERT_OK(output_stream->Close());
     }
@@ -297,8 +297,8 @@ TEST_F(RemoteLookupFileManagerTest, TryToDownloadLargeFileAcrossMultipleBuffers)
     {
         ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(local_file_path));
         std::vector read_buffer(file_size);
-        ASSERT_OK_AND_ASSIGN(int32_t bytes_read, input_stream->Read(read_buffer.data(), file_size));
-        ASSERT_EQ(static_cast(bytes_read), file_size);
+        ASSERT_OK_AND_ASSIGN(int64_t bytes_read, input_stream->Read(read_buffer.data(), file_size));
+        ASSERT_EQ(bytes_read, file_size);
         for (uint64_t i = 0; i < file_size; ++i) {
             ASSERT_EQ(read_buffer[i], static_cast(i % 251))
                 << "Data mismatch at byte offset " << i;
diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp
index 92eca73a..51bb5651 100644
--- a/src/paimon/core/mergetree/lookup_levels.cpp
+++ b/src/paimon/core/mergetree/lookup_levels.cpp
@@ -347,7 +347,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers,
         split_read_->CreateRawFileReaders(partition_, {file}, read_schema_,
                                           /*predicate=*/nullptr, dv_factory_,
-                                          /*row_ranges=*/std::nullopt, data_file_path_factory_));
+                                          /*row_ranges=*/std::nullopt, data_file_path_factory_,
+                                          /*extra_format_options=*/{}));
     if (raw_readers.size() != 1) {
         return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader");
     }
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index dbca52e0..3b6806c7 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -20,12 +20,14 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 
 #include "arrow/api.h"
 #include "arrow/c/abi.h"
 #include "arrow/c/helpers.h"
+#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/status_utils.h"
@@ -35,20 +37,17 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
-#include "paimon/core/io/key_value_data_file_writer.h"
+#include "paimon/core/io/key_value_data_file_writer_factory.h"
 #include "paimon/core/io/key_value_meta_projection_consumer.h"
 #include "paimon/core/io/key_value_record_reader.h"
 #include "paimon/core/io/row_to_arrow_array_converter.h"
-#include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h"
 #include "paimon/core/mergetree/write_buffer.h"
 #include "paimon/core/utils/commit_increment.h"
-#include "paimon/format/file_format.h"
-#include "paimon/format/writer_builder.h"
 
 namespace paimon {
-class FormatStatsExtractor;
 
 Result> MergeTreeWriter::Create(
     int64_t last_sequence_number, const std::vector& trimmed_primary_keys,
@@ -61,6 +60,7 @@ Result> MergeTreeWriter::Create(
     const std::shared_ptr& io_manager, bool enable_multi_thread_spill,
     const std::shared_ptr& pool) {
     auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
+
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr write_buffer,
         WriteBuffer::Create(last_sequence_number, value_schema, trimmed_primary_keys,
@@ -68,20 +68,20 @@ Result> MergeTreeWriter::Create(
                             merge_function_wrapper, options, io_manager, enable_multi_thread_spill,
                             pool));
     return std::shared_ptr(
-        new MergeTreeWriter(pool, trimmed_primary_keys, options, path_factory, key_comparator,
+        new MergeTreeWriter(trimmed_primary_keys, options, path_factory, key_comparator,
                             user_defined_seq_comparator, merge_function_wrapper, schema_id,
-                            write_schema, compact_manager, std::move(write_buffer)));
+                            write_schema, compact_manager, std::move(write_buffer), pool));
 }
 
 MergeTreeWriter::MergeTreeWriter(
-    const std::shared_ptr& pool, const std::vector& trimmed_primary_keys,
-    const CoreOptions& options, const std::shared_ptr& path_factory,
+    const std::vector& trimmed_primary_keys, const CoreOptions& options,
+    const std::shared_ptr& path_factory,
     const std::shared_ptr& key_comparator,
     const std::shared_ptr& user_defined_seq_comparator,
     const std::shared_ptr>& merge_function_wrapper,
     int64_t schema_id, const std::shared_ptr& write_schema,
     const std::shared_ptr& compact_manager,
-    std::unique_ptr&& write_buffer)
+    std::unique_ptr&& write_buffer, const std::shared_ptr& pool)
     : pool_(pool),
       trimmed_primary_keys_(trimmed_primary_keys),
       options_(options),
@@ -273,7 +273,9 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             std::make_unique>(
                 std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
                 /*projection_thread_num=*/1, pool_);
-        auto rolling_writer = CreateRollingRowWriter();
+        std::unique_ptr>>
+            rolling_writer;
+        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
         ScopeGuard write_guard([&]() -> void {
             rolling_writer->Abort();
             async_key_value_producer_consumer->Close();
@@ -317,35 +319,25 @@ Result MergeTreeWriter::DrainIncrement() {
     return CommitIncrement(data_increment, compact_increment, drain_deletion_file);
 }
 
-std::unique_ptr>>
+Result>>>
 MergeTreeWriter::CreateRollingRowWriter() const {
-    auto create_file_writer = [&]()
-        -> Result>>> {
-        ::ArrowSchema arrow_schema;
-        ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); });
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema));
-        auto format = options_.GetWriteFileFormat(/*level=*/0);
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr writer_builder,
-            format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize()));
-        writer_builder->WithMemoryPool(pool_);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema));
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor,
-                               format->CreateStatsExtractor(&arrow_schema));
-        auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status {
-            ArrowArrayMove(key_value_batch.batch.get(), array);
-            return Status::OK();
-        };
-        auto writer = std::make_unique(
-            options_.GetWriteFileCompression(0), converter, schema_id_, /*level=*/0,
-            FileSource::Append(), trimmed_primary_keys_, stats_extractor, write_schema_,
-            path_factory_->IsExternalPath(), pool_);
-        PAIMON_RETURN_NOT_OK(
-            writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder));
-        return writer;
-    };
+    std::shared_ptr>> factory;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
+            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory,
+            pool_);
+    } else {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
+            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_);
+    }
     return std::make_unique>>(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer);
+        options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
+        factory);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index bc1a1b49..febce2af 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -97,7 +97,7 @@ class MergeTreeWriter : public BatchWriter {
     Status FlushWriteBuffer(bool wait_for_latest_compaction, bool forced_full_compaction);
     Result DrainIncrement();
 
-    std::unique_ptr>>
+    Result>>>
     CreateRollingRowWriter() const;
 
     Status TrySyncLatestCompaction(bool blocking);
@@ -105,8 +105,7 @@ class MergeTreeWriter : public BatchWriter {
     Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file);
 
  private:
-    MergeTreeWriter(const std::shared_ptr& pool,
-                    const std::vector& trimmed_primary_keys,
+    MergeTreeWriter(const std::vector& trimmed_primary_keys,
                     const CoreOptions& options,
                     const std::shared_ptr& path_factory,
                     const std::shared_ptr& key_comparator,
@@ -114,7 +113,8 @@ class MergeTreeWriter : public BatchWriter {
                     const std::shared_ptr>& merge_function_wrapper,
                     int64_t schema_id, const std::shared_ptr& write_schema,
                     const std::shared_ptr& compact_manager,
-                    std::unique_ptr&& write_buffer);
+                    std::unique_ptr&& write_buffer,
+                    const std::shared_ptr& pool);
 
     std::shared_ptr pool_;
     std::vector trimmed_primary_keys_;
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index b4abb514..57a62654 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -31,6 +31,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -156,6 +157,32 @@ class MergeTreeWriterTest : public ::testing::TestWithParam {
         ASSERT_TRUE(expected_array->Equals(result_array)) << result_array->ToString();
     }
 
+    void CheckShreddingFileSchema(const std::string& data_file_name,
+                                  const std::shared_ptr& expected_physical_schema,
+                                  int32_t field_index,
+                                  const MapSharedShreddingFieldMeta& expected_meta) const {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(data_file_name));
+        ASSERT_TRUE(input_stream);
+        ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{}));
+        ASSERT_OK_AND_ASSIGN(auto reader_builder,
+                             file_format->CreateReaderBuilder(/*batch_size=*/10));
+        ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream));
+        ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema());
+        auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
+
+        ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false))
+            << "Expected schema:\n"
+            << expected_physical_schema->ToString() << "\nActual schema:\n"
+            << file_schema->ToString();
+
+        auto metadata = file_schema->field(field_index)->metadata();
+        ASSERT_NE(nullptr, metadata);
+        ASSERT_OK_AND_ASSIGN(auto deserialized_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
+        ASSERT_EQ(expected_meta, deserialized_meta);
+    }
+
     std::shared_ptr CreateMeta(const std::string& name, int32_t level) const {
         return std::make_shared(
             name, /*file_size=*/100, /*row_count=*/1, DataFileMeta::EmptyMinKey(),
@@ -349,6 +376,215 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) {
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
 }
 
+TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({
+                             {Options::FILE_FORMAT, "orc"},
+                             {"fields.tags.map.storage-layout", "shared-shredding"},
+                             {"fields.tags.map.shared-shredding.max-columns", "3"},
+                             {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+                             {Options::WRITE_ONLY, "true"},
+                         }));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+    std::string uuid = path_factory->uuid_;
+
+    std::vector value_fields = {
+        DataField(0, arrow::field("id", arrow::int32())),
+        DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))),
+    };
+    auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields);
+    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator,
+                         FieldsComparator::Create({value_fields[0]},
+                                                  /*is_ascending_order=*/true));
+    ASSERT_OK_AND_ASSIGN(
+        auto merge_writer,
+        MergeTreeWriter::Create(
+            /*last_sequence_number=*/9, /*trimmed_primary_keys=*/{"id"}, path_factory,
+            key_comparator,
+            /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5,
+            value_schema, options, noop_compact_manager_,
+            GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr,
+            /*enable_multi_thread_spill=*/false, pool_));
+
+    // Each batch contains duplicated primary keys. DeduplicateMergeFunction should keep the
+    // latest sequence number for each key across and within batches.
+    auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [1, [["a", 10], ["b", 20]]],
+      [2, [["c", 30]]],
+      [1, [["a", 11], ["c", 31]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array1, /*row_kinds=*/{}, merge_writer.get());
+
+    auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [2, [["b", 40]]],
+      [1, [["c", 50], ["d", 60]]],
+      [2, [["a", 70], ["b", 80], ["c", 90]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get());
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
+                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(merge_writer->Close());
+
+    ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty());
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+    auto actual_meta = commit_increment.GetNewFilesIncrement().NewFiles()[0];
+
+    std::string expected_data_file_name = "data-" + uuid + "-0.orc";
+    std::string expected_data_file_path = dir->Str() + "/" + expected_data_file_name;
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr data_file_status,
+                         options.GetFileSystem()->GetFileStatus(expected_data_file_path));
+
+    std::map column_to_k = {{"tags", 3}};
+    ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   write_schema, column_to_k));
+    auto physical_type = arrow::struct_(physical_schema->fields());
+
+    std::shared_ptr expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([
+      [14, 0, 1, [[0, 1, -1], 50, 60, null, null]],
+      [15, 0, 2, [[2, 3, 0], 70, 80, 90, null]]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(expected_data_file_path, expected_array);
+
+    MapSharedShreddingFieldMeta expected_shredding_meta;
+    expected_shredding_meta.name_to_id = {{"a", 2}, {"b", 3}, {"c", 0}, {"d", 1}};
+    expected_shredding_meta.field_to_columns = {{0, {0, 2}}, {1, {1}}, {2, {0}}, {3, {1}}};
+    expected_shredding_meta.num_columns = 3;
+    expected_shredding_meta.max_row_width = 3;
+    CheckShreddingFileSchema(expected_data_file_path, physical_schema, /*field_index=*/3,
+                             expected_shredding_meta);
+
+    auto expected_data_file_meta = std::make_shared(
+        expected_data_file_name, /*file_size=*/data_file_status->GetLen(), /*row_count=*/2,
+        /*min_key=*/BinaryRowGenerator::GenerateRow({1}, pool_.get()),
+        /*max_key=*/BinaryRowGenerator::GenerateRow({2}, pool_.get()),
+        /*key_stats=*/
+        BinaryRowGenerator::GenerateStats({1}, {2}, {0}, pool_.get()),
+        /*value_stats=*/
+        BinaryRowGenerator::GenerateStats({1, NullType()}, {2, NullType()}, {0, 0}, pool_.get()),
+        /*min_sequence_number=*/14, /*max_sequence_number=*/15, /*schema_id=*/5,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/actual_meta->creation_time, /*delete_row_count=*/0,
+        /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt);
+    ASSERT_TRUE(expected_data_file_meta->TEST_Equal(*actual_meta));
+}
+
+TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsAdaptAcrossRollingFiles) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({
+                             {Options::FILE_FORMAT, "orc"},
+                             {Options::TARGET_FILE_ROW_NUM, "2"},
+                             {Options::WRITE_BATCH_SIZE, "2"},
+                             {"fields.tags.map.storage-layout", "shared-shredding"},
+                             {"fields.tags.map.shared-shredding.max-columns", "8"},
+                             {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+                             {"fields.attrs.map.storage-layout", "shared-shredding"},
+                             {"fields.attrs.map.shared-shredding.max-columns", "4"},
+                             {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"},
+                             {Options::WRITE_ONLY, "true"},
+                         }));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared();
+    ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr));
+
+    std::vector value_fields = {
+        DataField(0, arrow::field("id", arrow::int32())),
+        DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))),
+        DataField(2, arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8()))),
+    };
+    auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields);
+    auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields);
+    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator,
+                         FieldsComparator::Create({value_fields[0]},
+                                                  /*is_ascending_order=*/true));
+    ASSERT_OK_AND_ASSIGN(
+        auto merge_writer,
+        MergeTreeWriter::Create(
+            /*last_sequence_number=*/-1, /*trimmed_primary_keys=*/{"id"}, path_factory,
+            key_comparator,
+            /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0,
+            value_schema, options, noop_compact_manager_,
+            GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr,
+            /*enable_multi_thread_spill=*/false, pool_));
+
+    auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [1, [["a", 10], ["b", 20]], [["x", "v1"]]],
+      [2, [["a", 30]],           [["x", "v2"]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array1, /*row_kinds=*/{}, merge_writer.get());
+
+    auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get());
+
+    auto array3 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array3, /*row_kinds=*/{}, merge_writer.get());
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+    std::string file1_path = path_factory->ToPath(files[0]->file_name);
+    std::string file2_path = path_factory->ToPath(files[1]->file_name);
+
+    std::map column_to_k_file1 = {{"tags", 8}, {"attrs", 4}};
+    ASSERT_OK_AND_ASSIGN(auto physical_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                    write_schema, column_to_k_file1));
+    MapSharedShreddingFieldMeta tags_meta1;
+    tags_meta1.name_to_id = {{"a", 0}, {"b", 1}};
+    tags_meta1.field_to_columns = {{0, {0}}, {1, {1}}};
+    tags_meta1.num_columns = 8;
+    tags_meta1.max_row_width = 2;
+    CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/3, tags_meta1);
+    MapSharedShreddingFieldMeta attrs_meta1;
+    attrs_meta1.name_to_id = {{"x", 0}};
+    attrs_meta1.field_to_columns = {{0, {0}}};
+    attrs_meta1.num_columns = 4;
+    attrs_meta1.max_row_width = 1;
+    CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/4, attrs_meta1);
+
+    std::map column_to_k_file2 = {{"tags", 2}, {"attrs", 1}};
+    ASSERT_OK_AND_ASSIGN(auto physical_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                    write_schema, column_to_k_file2));
+    MapSharedShreddingFieldMeta tags_meta2;
+    tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}, {"f", 3}, {"g", 4}};
+    tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}, {3, {0}}, {4, {1}}};
+    tags_meta2.overflow_field_set = {2};
+    tags_meta2.num_columns = 2;
+    tags_meta2.max_row_width = 3;
+    CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/3, tags_meta2);
+    MapSharedShreddingFieldMeta attrs_meta2;
+    attrs_meta2.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}, {"s", 3}, {"t", 4}};
+    attrs_meta2.field_to_columns = {{0, {0}}, {3, {0}}};
+    attrs_meta2.overflow_field_set = {1, 2, 4};
+    attrs_meta2.num_columns = 1;
+    attrs_meta2.max_row_width = 3;
+    CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/4, attrs_meta2);
+
+    ASSERT_OK(merge_writer->Close());
+}
+
 TEST_P(MergeTreeWriterTest, TestWriteWithDeleteRow) {
     ASSERT_OK_AND_ASSIGN(
         CoreOptions options,
diff --git a/src/paimon/core/mergetree/row_count_accumulator.cpp b/src/paimon/core/mergetree/row_count_accumulator.cpp
new file mode 100644
index 00000000..8cbc04fb
--- /dev/null
+++ b/src/paimon/core/mergetree/row_count_accumulator.cpp
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/mergetree/row_count_accumulator.h"
+
+#include 
+
+#include "paimon/core/key_value.h"
+
+namespace paimon {
+
+RowCountAccumulator::RowCountAccumulator(std::unique_ptr&& merged_reader)
+    : merged_reader_(std::move(merged_reader)) {}
+
+Result RowCountAccumulator::CountAll() {
+    int64_t count = 0;
+
+    while (true) {
+        // Get next batch of merged KV iterators
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter,
+                               merged_reader_->NextBatch());
+        if (iter == nullptr) {
+            // No more data
+            break;
+        }
+
+        // Iterate through all KV objects in this batch
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(bool has_next, iter->HasNext());
+            if (!has_next) {
+                break;
+            }
+
+            iter->Next();
+
+            // At this point:
+            // - kv has passed through SortMergeReader (deduplicated, merged)
+            // - kv has passed through DropDeleteReader (kind is guaranteed IsAdd())
+            // - kv represents a final, valid, non-deleted row
+            count++;
+        }
+    }
+
+    return count;
+}
+
+void RowCountAccumulator::Close() {
+    if (merged_reader_) {
+        merged_reader_->Close();
+    }
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/mergetree/row_count_accumulator.h b/src/paimon/core/mergetree/row_count_accumulator.h
new file mode 100644
index 00000000..7e5bf98e
--- /dev/null
+++ b/src/paimon/core/mergetree/row_count_accumulator.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+
+#include "paimon/core/mergetree/compact/sort_merge_reader.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+/// Counts rows from a merged KeyValue stream after delete rows are dropped.
+class RowCountAccumulator {
+ public:
+    /// @param merged_reader  The merged reader. Must be wrapped with DropDeleteReader
+    ///                       so that only valid (non-deleted) KeyValue objects are output.
+    explicit RowCountAccumulator(std::unique_ptr&& merged_reader);
+
+    ~RowCountAccumulator() = default;
+
+    /// Count all valid rows from the merge reader.
+    /// Iterates through all merged+deduplicated+non-deleted KeyValue objects.
+    Result CountAll();
+
+    /// Close underlying readers and release resources.
+    void Close();
+
+ private:
+    std::unique_ptr merged_reader_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/mergetree/spill_reader.cpp b/src/paimon/core/mergetree/spill_reader.cpp
index 0b338c29..e1ed9004 100644
--- a/src/paimon/core/mergetree/spill_reader.cpp
+++ b/src/paimon/core/mergetree/spill_reader.cpp
@@ -54,9 +54,9 @@ Status SpillReader::Open(const FileIOChannel::ID& channel_id) {
     const std::string& file_path = channel_id.GetPath();
     PAIMON_ASSIGN_OR_RAISE(in_stream_, fs_->Open(file_path));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(file_path));
-    uint64_t file_len = file_status->GetLen();
+    int64_t file_len = file_status->GetLen();
     arrow_input_stream_adapter_ =
-        std::make_shared(in_stream_, arrow_pool_, file_len);
+        std::make_shared(in_stream_, file_len, arrow_pool_);
     auto ipc_read_options = arrow::ipc::IpcReadOptions::Defaults();
     ipc_read_options.memory_pool = arrow_pool_.get();
     ipc_read_options.use_threads = use_threads_;
@@ -78,7 +78,7 @@ Result SpillReader::Iterator::Next() {
     PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
                            RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));
     int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_);
-    auto key = std::make_unique(reader_->key_ctx_, cursor_);
+    std::shared_ptr key = std::make_shared(reader_->key_ctx_, cursor_);
     auto value = std::make_unique(reader_->value_ctx_, cursor_);
     cursor_++;
     return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key),
diff --git a/src/paimon/core/mergetree/spill_writer.cpp b/src/paimon/core/mergetree/spill_writer.cpp
index 3e9b917c..170057fc 100644
--- a/src/paimon/core/mergetree/spill_writer.cpp
+++ b/src/paimon/core/mergetree/spill_writer.cpp
@@ -117,7 +117,7 @@ Result SpillWriter::GetFileSize() const {
     }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status,
                            fs_->GetFileStatus(channel_id_.GetPath()));
-    return static_cast(file_status->GetLen());
+    return file_status->GetLen();
 }
 
 const FileIOChannel::ID& SpillWriter::GetChannelId() const {
diff --git a/src/paimon/core/mergetree/write_buffer_test.cpp b/src/paimon/core/mergetree/write_buffer_test.cpp
index 0f082737..26f02933 100644
--- a/src/paimon/core/mergetree/write_buffer_test.cpp
+++ b/src/paimon/core/mergetree/write_buffer_test.cpp
@@ -102,7 +102,7 @@ class WriteBufferTest : public ::testing::Test {
         if (spill_files.size() != 1 || spill_files[0]->IsDir()) {
             return Status::Invalid("expected exactly one spill file");
         }
-        return static_cast(spill_files[0]->GetLen());
+        return spill_files[0]->GetLen();
     }
 
     Result ReadReaderResult(KeyValueRecordReader* reader) const {
diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp
index 24a5363e..d7dafa18 100644
--- a/src/paimon/core/operation/abstract_file_store_write.cpp
+++ b/src/paimon/core/operation/abstract_file_store_write.cpp
@@ -76,7 +76,7 @@ AbstractFileStoreWrite::AbstractFileStoreWrite(
       dv_maintainer_factory_(dv_maintainer_factory),
       io_manager_(io_manager),
       options_(options),
-      compact_executor_(CreateDefaultExecutor(4)),
+      compact_executor_(CreateDefaultExecutor()),
       compaction_metrics_(std::make_shared()),
       ignore_previous_files_(ignore_previous_files),
       is_streaming_mode_(is_streaming_mode),
@@ -128,7 +128,7 @@ Status AbstractFileStoreWrite::Write(std::unique_ptr&& batch) {
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data, batch->GetData()));
 
     PAIMON_ASSIGN_OR_RAISE(BinaryRow partition,
-                           file_store_path_factory_->ToBinaryRow(batch->GetPartition()))
+                           file_store_path_factory_->ToBinaryRow(batch->GetPartition()));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer,
                            GetWriter(partition, batch->GetBucket()));
     assert(writer);
@@ -283,7 +283,8 @@ int32_t AbstractFileStoreWrite::GetDefaultBucketNum() const {
 
 Result> AbstractFileStoreWrite::ScanExistingFileMetas(
     const BinaryRow& partition, int32_t bucket) const {
-    PAIMON_ASSIGN_OR_RAISE(auto part_values,
+    std::vector> part_values;
+    PAIMON_ASSIGN_OR_RAISE(part_values,
                            file_store_path_factory_->GeneratePartitionVector(partition));
     std::map part_values_map;
     for (const auto& [key, value] : part_values) {
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 30b2215a..0850d943 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -20,9 +20,19 @@
 
 #include 
 #include 
+#include 
+#include 
 #include 
 
 #include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/data/blob_defs.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/shredding_file_reader.h"
+#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/reader/delegating_prefetch_reader.h"
 #include "paimon/common/reader/predicate_batch_reader.h"
 #include "paimon/common/reader/prefetch_file_batch_reader_impl.h"
@@ -38,6 +48,7 @@
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/format/file_format.h"
 #include "paimon/format/file_format_factory.h"
 #include "paimon/fs/file_system.h"
@@ -67,7 +78,8 @@ Result>> AbstractSplitRead::CreateR
     const BinaryRow& partition, const std::vector>& data_files,
     const std::shared_ptr& read_schema, const std::shared_ptr& predicate,
     DeletionVector::Factory dv_factory, const std::optional>& row_ranges,
-    const std::shared_ptr& data_file_path_factory) const {
+    const std::shared_ptr& data_file_path_factory,
+    const std::map& extra_format_options) const {
     if (data_files.empty()) {
         return std::vector>();
     }
@@ -81,7 +93,7 @@ Result>> AbstractSplitRead::CreateR
         auto data_file_path = data_file_path_factory->ToPath(file);
         PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat());
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
-                               PrepareReaderBuilder(data_file_identifier));
+                               PrepareReaderBuilder(data_file_identifier, extra_format_options));
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr file_reader,
             CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(),
@@ -103,29 +115,6 @@ bool AbstractSplitRead::NeedCompleteRowTrackingFields(
     }
     return false;
 }
-
-std::unordered_map AbstractSplitRead::CreateDeletionFileMap(
-    const DataSplitImpl& data_split) {
-    return CreateDeletionFileMap(data_split.DataFiles(), data_split.DeletionFiles());
-}
-
-std::unordered_map AbstractSplitRead::CreateDeletionFileMap(
-    const std::vector>& data_files,
-    const std::vector>& deletion_files) {
-    std::unordered_map deletion_file_map;
-    if (deletion_files.empty()) {
-        return deletion_file_map;
-    }
-    assert(deletion_files.size() == data_files.size());
-    size_t file_count = deletion_files.size();
-    for (size_t i = 0; i < file_count; i++) {
-        if (deletion_files[i] != std::nullopt) {
-            deletion_file_map.emplace(data_files[i]->file_name, deletion_files[i].value());
-        }
-    }
-    return deletion_file_map;
-}
-
 Result> AbstractSplitRead::ApplyPredicateFilterIfNeeded(
     std::unique_ptr&& reader, const std::shared_ptr& predicate) const {
     if (!context_->EnablePredicateFilter() || predicate == nullptr) {
@@ -135,19 +124,27 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe
 }
 
 Result> AbstractSplitRead::PrepareReaderBuilder(
-    const std::string& format_identifier) const {
+    const std::string& format_identifier,
+    const std::map& extra_format_options) const {
+    std::map format_options = options_.ToMap();
+    // The blob placeholder channels are internal: strip user-supplied blob.internal.* table
+    // options so only the internal read path can enable them through extra_format_options.
+    BlobDefs::EraseInternalPlaceholderOptions(&format_options);
+    for (const auto& [key, value] : extra_format_options) {
+        format_options[key] = value;
+    }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format,
-                           FileFormatFactory::Get(format_identifier, options_.ToMap()));
+                           FileFormatFactory::Get(format_identifier, format_options));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
                            file_format->CreateReaderBuilder(options_.GetReadBatchSize()));
     reader_builder->WithMemoryPool(pool_);
+    reader_builder->WithCache(options_.GetCache());
     return reader_builder;
 }
 
 Result> AbstractSplitRead::CreateFileBatchReader(
-    const std::shared_ptr& file_meta, const std::string& data_file_path,
+    const std::string& file_format_identifier, const std::string& data_file_path,
     const ReaderBuilder* reader_builder) const {
-    PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat());
     if (context_->EnablePrefetch() && file_format_identifier != "blob" &&
         file_format_identifier != "avro") {
         PAIMON_ASSIGN_OR_RAISE(
@@ -180,6 +177,10 @@ Result> AbstractSplitRead::CreateFieldMappingRe
         // load schema to get data schema
         PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(file_meta->schema_id));
     }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions data_options,
+                           CoreOptions::FromMap(data_schema->Options(), options_.GetFileSystem()));
+    auto blob_inline_fields = data_options.GetBlobInlineFields();
+
     std::unique_ptr field_mapping;
     if (!data_schema->PrimaryKeys().empty()) {
         // for pk table, add special fields to file schema when field mapping
@@ -193,20 +194,40 @@ Result> AbstractSplitRead::CreateFieldMappingRe
         PAIMON_ASSIGN_OR_RAISE(
             std::vector projected_data_fields,
             ProjectFieldsForRowTrackingAndDataEvolution(data_schema, file_meta->write_cols));
+        auto converted_fields =
+            BlobUtils::ConvertBlobInlineDataFields(projected_data_fields, blob_inline_fields);
         PAIMON_ASSIGN_OR_RAISE(field_mapping,
-                               field_mapping_builder->CreateFieldMapping(projected_data_fields));
+                               field_mapping_builder->CreateFieldMapping(converted_fields));
     }
 
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(
         field_mapping->non_partition_info.non_partition_data_schema);
 
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_reader,
-                           CreateFileBatchReader(file_meta, data_file_path, reader_builder));
+    PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr file_reader,
+        CreateFileBatchReader(file_format_identifier, data_file_path, reader_builder));
+    std::set skip_map_selected_keys_filter_field_ids;
+    if (file_format_identifier != "blob") {
+        std::pair, std::set> shared_shredding_result;
+        PAIMON_ASSIGN_OR_RAISE(shared_shredding_result, ApplySharedShreddingReaderIfNeeded(
+                                                            std::move(file_reader), read_schema));
+        file_reader = std::move(shared_shredding_result.first);
+        skip_map_selected_keys_filter_field_ids = std::move(shared_shredding_result.second);
+        PAIMON_ASSIGN_OR_RAISE(
+            file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema));
+    }
     if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) {
+        // A blob file has no self-describing schema: its physical fields are declared by the
+        // file meta's write cols instead of queried from the format reader.
+        std::optional> file_field_names;
+        if (file_format_identifier == "blob") {
+            file_field_names = file_meta->write_cols;
+        }
         file_reader = std::make_unique(
-            std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, pool_);
+            std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number,
+            file_field_names, pool_);
     }
-
     const auto& predicate = field_mapping->non_partition_info.non_partition_filter;
     auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields());
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr final_reader,
@@ -218,9 +239,94 @@ Result> AbstractSplitRead::CreateFieldMappingRe
         return std::unique_ptr();
     }
 
-    return std::make_unique(field_mapping_builder->GetReadFieldCount(),
-                                                std::move(final_reader), partition,
-                                                std::move(field_mapping), pool_);
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr mapping_reader,
+        FieldMappingReader::Create(field_mapping_builder->GetReadFieldCount(),
+                                   std::move(final_reader), partition, std::move(field_mapping),
+                                   std::move(skip_map_selected_keys_filter_field_ids), pool_));
+    return mapping_reader;
+}
+
+Result, std::set>>
+AbstractSplitRead::ApplySharedShreddingReaderIfNeeded(
+    std::unique_ptr&& file_reader,
+    const std::shared_ptr& read_schema) const {
+    std::set handled_shared_shredding_field_ids;
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema,
+                           file_reader->GetFileSchema());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema,
+                                      arrow::ImportSchema(file_schema.get()));
+    std::map
+        shared_shredding_name_to_context;
+    for (const auto& read_field : read_schema->fields()) {
+        const auto& field_name = read_field->name();
+        auto file_field = file_arrow_schema->GetFieldByName(field_name);
+        if (!file_field) {
+            // may exists field _ROW_ID in read schema
+            continue;
+        }
+        std::shared_ptr metadata =
+            std::const_pointer_cast(file_field->metadata());
+        if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) {
+            // not a map shared shredding field
+            continue;
+        }
+        // get meta
+        PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta,
+                               MapSharedShreddingUtils::DeserializeMetadata(metadata));
+        // get selected_keys
+        PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys,
+                               NestedProjectionUtils::GetMapSelectedKeys(read_field));
+        if (selected_keys.empty()) {
+            // select all keys
+            selected_keys.reserve(meta.name_to_id.size());
+            for (const auto& [key_name, _] : meta.name_to_id) {
+                selected_keys.push_back(key_name);
+            }
+        }
+        // get map type
+        auto map_type = arrow::internal::checked_pointer_cast(read_field->type());
+        shared_shredding_name_to_context.emplace(
+            field_name,
+            MapSharedShreddingFileReader::SharedShreddingContext(meta, selected_keys, map_type));
+        PAIMON_ASSIGN_OR_RAISE(int32_t field_id,
+                               NestedProjectionUtils::GetPaimonFieldId(read_field));
+        handled_shared_shredding_field_ids.insert(field_id);
+    }
+    if (!shared_shredding_name_to_context.empty()) {
+        file_reader = std::make_unique(
+            std::move(file_reader), std::move(shared_shredding_name_to_context), pool_);
+    }
+    return std::make_pair(std::move(file_reader), std::move(handled_shared_shredding_field_ids));
+}
+
+Result> AbstractSplitRead::ApplyVariantShreddingReaderIfNeeded(
+    std::unique_ptr&& file_reader,
+    const std::shared_ptr& read_schema) const {
+    bool has_variant_field = false;
+    for (const auto& read_field : read_schema->fields()) {
+        // Variant columns may be nested inside struct columns; a variant-access projection also
+        // matches because it carries the variant extension marker itself.
+        if (VariantTypeUtils::ContainsVariantField(read_field)) {
+            has_variant_field = true;
+            break;
+        }
+    }
+    if (!has_variant_field) {
+        return std::move(file_reader);
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema,
+                           file_reader->GetFileSchema());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema,
+                                      arrow::ImportSchema(file_schema.get()));
+    std::map> plans;
+    PAIMON_ASSIGN_OR_RAISE(plans, VariantShreddingReadPlanFactory::CreateReadPlans(
+                                      read_schema, file_arrow_schema, pool_));
+    if (!plans.empty()) {
+        file_reader =
+            std::make_unique(std::move(file_reader), std::move(plans), pool_);
+    }
+    return std::move(file_reader);
 }
 
 Result> AbstractSplitRead::ProjectFieldsForRowTrackingAndDataEvolution(
diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h
index ed5dda98..ea3f9070 100644
--- a/src/paimon/core/operation/abstract_split_read.h
+++ b/src/paimon/core/operation/abstract_split_read.h
@@ -18,10 +18,12 @@
 
 #pragma once
 
+#include 
 #include 
 #include 
+#include 
 #include 
-#include 
+#include 
 #include 
 
 #include "arrow/type_fwd.h"
@@ -32,7 +34,6 @@
 #include "paimon/core/operation/split_read.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/table/source/data_split_impl.h"
-#include "paimon/core/table/source/deletion_file.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/format/reader_builder.h"
 #include "paimon/reader/batch_reader.h"
@@ -62,12 +63,16 @@ class AbstractSplitRead : public SplitRead {
  public:
     ~AbstractSplitRead() override = default;
 
+    /// `extra_format_options` are merged over the table options when building the format
+    /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the
+    /// data-evolution blob fallback read path.
     Result>> CreateRawFileReaders(
         const BinaryRow& partition, const std::vector>& data_files,
         const std::shared_ptr& read_schema,
         const std::shared_ptr& predicate, DeletionVector::Factory dv_factory,
         const std::optional>& row_ranges,
-        const std::shared_ptr& data_file_path_factory) const;
+        const std::shared_ptr& data_file_path_factory,
+        const std::map& extra_format_options) const;
 
  protected:
     AbstractSplitRead(const std::shared_ptr& path_factory,
@@ -75,14 +80,6 @@ class AbstractSplitRead : public SplitRead {
                       std::unique_ptr&& schema_manager,
                       const std::shared_ptr& memory_pool,
                       const std::shared_ptr& executor);
-
-    static std::unordered_map CreateDeletionFileMap(
-        const DataSplitImpl& data_split);
-
-    static std::unordered_map CreateDeletionFileMap(
-        const std::vector>& data_files,
-        const std::vector>& deletion_files);
-
     Result> ApplyPredicateFilterIfNeeded(
         std::unique_ptr&& reader, const std::shared_ptr& predicate) const;
 
@@ -105,10 +102,11 @@ class AbstractSplitRead : public SplitRead {
 
  private:
     Result> PrepareReaderBuilder(
-        const std::string& format_identifier) const;
+        const std::string& format_identifier,
+        const std::map& extra_format_options) const;
 
     Result> CreateFileBatchReader(
-        const std::shared_ptr& file_meta, const std::string& data_file_path,
+        const std::string& file_format_identifier, const std::string& data_file_path,
         const ReaderBuilder* reader_builder) const;
 
     // return nullptr if data file is skipped by index or dv
@@ -119,6 +117,17 @@ class AbstractSplitRead : public SplitRead {
         const std::optional>& row_ranges,
         const std::shared_ptr& data_file_path_factory) const;
 
+    Result, std::set>>
+    ApplySharedShreddingReaderIfNeeded(std::unique_ptr&& file_reader,
+                                       const std::shared_ptr& read_schema) const;
+
+    /// Wraps the reader with a `ShreddingFileReader` when any read variant column needs
+    /// reassembly or path extraction; a plain read of an unshredded variant column is passed
+    /// through untouched.
+    Result> ApplyVariantShreddingReaderIfNeeded(
+        std::unique_ptr&& file_reader,
+        const std::shared_ptr& read_schema) const;
+
     static bool NeedCompleteRowTrackingFields(bool row_tracking_enabled,
                                               const std::shared_ptr& read_schema);
 
diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
index 3bdf112e..1fe75338 100644
--- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp
@@ -20,6 +20,7 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -27,6 +28,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/io/cache/lru_cache.h"
 #include "paimon/core/manifest/partition_entry.h"
 #include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/schema/schema_manager.h"
@@ -84,6 +86,12 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) {
         auto result,
         AppendOnlyFileStoreScan::ReconstructPredicateWithNonCastedFields(predicate, evo));
     ASSERT_EQ(*result, *child4);
+
+    auto key_predicate =
+        PredicateBuilder::IsNull(/*field_index=*/1, /*field_name=*/"key0", FieldType::INT);
+    ASSERT_OK_AND_ASSIGN(result, AppendOnlyFileStoreScan::ReconstructPredicateWithNonCastedFields(
+                                     key_predicate, evo));
+    ASSERT_EQ(*result, *key_predicate);
 }
 
 TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) {
@@ -171,4 +179,72 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) {
     ASSERT_LE(stats.p50, stats.p99);
     ASSERT_LE(stats.p99, stats.max);
 }
+
+namespace {
+
+std::shared_ptr BuildScan(const std::string& table_path,
+                                         const std::shared_ptr& cache,
+                                         const std::optional& bucket = std::nullopt) {
+    ScanContextBuilder context_builder(table_path);
+    context_builder.AddOption(Options::FILE_FORMAT, "orc")
+        .AddOption(Options::MANIFEST_FORMAT, "orc")
+        .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "8")
+        .WithCache(cache);
+    if (bucket) {
+        context_builder.SetBucketFilter(bucket.value());
+    }
+    EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish());
+    EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context)));
+    auto typed_table_scan = dynamic_cast(table_scan.get());
+    EXPECT_TRUE(typed_table_scan);
+    return typed_table_scan->snapshot_reader_->scan_;
+}
+
+}  // namespace
+
+TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCachePath) {
+    TimezoneGuard guard("Asia/Shanghai");
+    std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+    auto cache = std::make_shared(/*max_weight=*/16 * 1024 * 1024);
+
+    // First scan on snapshot 5: cache miss, entries rebuilt from all manifests.
+    auto scan_first = BuildScan(table_path, cache, /*bucket=*/0);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot_5,
+                         scan_first->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5));
+    scan_first->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_first, scan_first->CreatePlan());
+    size_t first_size = plan_first->Files().size();
+
+    // Second scan on the same snapshot should read the same bucket live entries from cache.
+    auto scan_second = BuildScan(table_path, cache, /*bucket=*/0);
+    scan_second->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_second, scan_second->CreatePlan());
+    ASSERT_EQ(first_size, plan_second->Files().size());
+}
+
+TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheRebuildOnMiss) {
+    TimezoneGuard guard("Asia/Shanghai");
+    std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
+    auto cache = std::make_shared(/*max_weight=*/16 * 1024 * 1024);
+
+    // Seed the cache with an earlier snapshot.
+    auto scan_base = BuildScan(table_path, cache, /*bucket=*/0);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3,
+                         scan_base->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/3));
+    scan_base->WithSnapshot(snapshot_3);
+    ASSERT_OK_AND_ASSIGN(auto plan_base, scan_base->CreatePlan());
+    (void)plan_base;
+
+    // Advance to a newer snapshot: cache miss rebuilds the target snapshot bucket.
+    auto scan_next = BuildScan(table_path, cache, /*bucket=*/0);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot_5,
+                         scan_next->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5));
+    scan_next->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_next, scan_next->CreatePlan());
+
+    auto scan_expected = BuildScan(table_path, /*cache=*/nullptr, /*bucket=*/0);
+    scan_expected->WithSnapshot(snapshot_5);
+    ASSERT_OK_AND_ASSIGN(auto plan_expected, scan_expected->CreatePlan());
+    ASSERT_EQ(plan_expected->Files().size(), plan_next->Files().size());
+}
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index 427a0fa0..135c49dc 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -18,19 +18,25 @@
 
 #include "paimon/core/operation/append_only_file_store_write.h"
 
+#include 
+#include 
 #include 
 
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
 #include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/core/append/append_only_writer.h"
 #include "paimon/core/append/bucketed_append_compact_manager.h"
 #include "paimon/core/compact/noop_compact_manager.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/io/append_data_file_writer_factory.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/io/data_file_path_factory.h"
-#include "paimon/core/io/data_file_writer.h"
 #include "paimon/core/io/rolling_file_writer.h"
+#include "paimon/core/io/shredding_append_data_file_writer_factory.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/operation/append_only_file_store_scan.h"
@@ -73,10 +79,6 @@ AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite(
                              is_streaming_mode, ignore_num_bucket_check, executor, pool),
       logger_(Logger::GetLogger("AppendOnlyFileStoreWrite")) {
     write_cols_ = write_schema->field_names();
-    auto schemas = BlobUtils::SeparateBlobSchema(schema_);
-    if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) {
-        with_blob_ = true;
-    }
     // optimize write_cols to null in following cases:
     // 1. write_schema contains all columns
     // 2. TODO(xinyu.lxy) write_schema contains all columns and append _ROW_ID & _SEQUENCE_NUMBER
@@ -93,7 +95,8 @@ Result> AppendOnlyFileStoreWrite::CreateFileStore
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_list,
         ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
-                             options_.GetManifestCompression(), file_store_path_factory_, pool_));
+                             options_.GetManifestCompression(), file_store_path_factory_,
+                             options_.GetCache(), pool_));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_file,
         ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
@@ -117,10 +120,15 @@ Result>> AppendOnlyFileStoreWrite::Com
 
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
                            CreateFilesReader(partition, bucket, dv_factory, to_compact));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
+                           file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
+    PAIMON_ASSIGN_OR_RAISE(
+        WriterFactory writer_factory,
+        GetDataFileWriterFactory(data_file_path_factory, write_schema_, write_cols_, to_compact));
     auto rewriter =
         std::make_unique>>(
             options_.GetTargetFileSize(/*has_primary_key=*/false),
-            GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact));
+            /*target_file_row_num=*/std::numeric_limits::max(), writer_factory);
 
     ScopeGuard reader_guard([&]() {
         if (reader) {
@@ -174,9 +182,7 @@ Result> AppendOnlyFileStoreWrite::CreateWriter(
                            file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
 
     std::shared_ptr compact_manager;
-    auto schemas = BlobUtils::SeparateBlobSchema(write_schema_);
-    if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1 ||
-        with_blob_) {
+    if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1) {
         compact_manager = std::make_shared();
     } else {
         auto dv_factory =
@@ -206,46 +212,28 @@ Result> AppendOnlyFileStoreWrite::CreateWriter(
             compaction_metrics_->CreateReporter(partition, bucket), cancellation_controller);
     }
 
-    auto writer = std::make_shared(
+    auto writer = std::make_unique(
         options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number,
         data_file_path_factory, compact_manager, pool_);
-    return std::shared_ptr(writer);
+    return std::shared_ptr(std::move(writer));
 }
 
-AppendOnlyFileStoreWrite::SingleFileWriterCreator
-AppendOnlyFileStoreWrite::GetDataFileWriterCreator(
-    const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema,
+Result AppendOnlyFileStoreWrite::GetDataFileWriterFactory(
+    const std::shared_ptr& data_file_path_factory,
+    const std::shared_ptr& schema,
     const std::optional>& write_cols,
     const std::vector>& to_compact) const {
-    return
-        [this, partition, bucket, schema, write_cols, to_compact]()
-            -> Result<
-                std::unique_ptr>>> {
-            ::ArrowSchema arrow_schema;
-            ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); });
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema));
-            auto format = options_.GetFileFormat();
-            PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr writer_builder,
-                format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize()));
-            writer_builder->WithMemoryPool(pool_);
-
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema));
-            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor,
-                                   format->CreateStatsExtractor(&arrow_schema));
-            PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr data_file_path_factory,
-                file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
-            auto writer = std::make_unique(
-                options_.GetFileCompression(), std::function(),
-                table_schema_->Id(),
-                std::make_shared(to_compact[0]->min_sequence_number),
-                FileSource::Compact(), stats_extractor, data_file_path_factory->IsExternalPath(),
-                write_cols, pool_);
-            PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(),
-                                              data_file_path_factory->NewPath(), writer_builder));
-            return writer;
-        };
+    auto seq_num_counter = std::make_shared(to_compact[0]->min_sequence_number);
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan_factory,
+                           ShreddingWritePlanFactories::SelectActive(options_, schema, pool_));
+    if (plan_factory != nullptr) {
+        return std::make_shared(
+            options_, table_schema_->Id(), schema, write_cols, seq_num_counter,
+            FileSource::Compact(), data_file_path_factory, plan_factory, pool_);
+    }
+    return std::make_shared(
+        options_, table_schema_->Id(), schema, write_cols, seq_num_counter, FileSource::Compact(),
+        data_file_path_factory, pool_);
 }
 
 Result> AppendOnlyFileStoreWrite::CreateFilesReader(
diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h
index d1aacb5a..905a4180 100644
--- a/src/paimon/core/operation/append_only_file_store_write.h
+++ b/src/paimon/core/operation/append_only_file_store_write.h
@@ -31,7 +31,6 @@
 #include "paimon/core/compact/cancellation_controller.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/deletionvectors/deletion_vector.h"
-#include "paimon/core/io/single_file_writer.h"
 #include "paimon/core/operation/abstract_file_store_write.h"
 #include "paimon/core/table/bucket_mode.h"
 #include "paimon/file_store_write.h"
@@ -41,6 +40,7 @@
 #include "paimon/type_fwd.h"
 
 struct ArrowSchema;
+struct ArrowArray;
 
 namespace arrow {
 class Schema;
@@ -51,10 +51,13 @@ namespace paimon {
 struct DataFileMeta;
 class BatchWriter;
 class BucketedDvMaintainer;
+class DataFilePathFactory;
 class FileStorePathFactory;
 class FileStoreScan;
 class SnapshotManager;
 class ScanFilter;
+template 
+class SingleFileWriterFactory;
 class MetricsImpl;
 class BinaryRow;
 class CoreOptions;
@@ -95,8 +98,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite {
         const std::shared_ptr& cancellation_controller);
 
  private:
-    using SingleFileWriterCreator = std::function<
-        Result>>>()>;
+    using WriterFactory =
+        std::shared_ptr>>;
 
     Result> CreateWriter(
         const BinaryRow& partition, int32_t bucket,
@@ -107,8 +110,9 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite {
     Result> CreateFileStoreScan(
         const std::shared_ptr& filter) const override;
 
-    SingleFileWriterCreator GetDataFileWriterCreator(
-        const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema,
+    Result GetDataFileWriterFactory(
+        const std::shared_ptr& data_file_path_factory,
+        const std::shared_ptr& schema,
         const std::optional>& write_cols,
         const std::vector>& to_compact) const;
 
@@ -117,7 +121,6 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite {
         const std::vector>& files) const;
 
     std::optional> write_cols_;
-    bool with_blob_ = false;
     std::unique_ptr logger_;
 };
 
diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp
index 09b9e28a..df1e1bd2 100644
--- a/src/paimon/core/operation/append_only_file_store_write_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp
@@ -20,6 +20,8 @@
 
 #include 
 #include 
+#include 
+#include 
 #include 
 
 #include "arrow/array/array_base.h"
@@ -27,20 +29,31 @@
 #include "arrow/c/abi.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "arrow/ipc/json_simple.h"
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "gtest/gtest.h"
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/restore_files.h"
 #include "paimon/core/snapshot.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
 #include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/reader/file_batch_reader.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/testharness.h"
@@ -68,6 +81,101 @@ class AppendOnlyFileStoreWriteTest : public testing::Test {
         commit_user_ = "test_commit_user";
     }
 
+    void CreateTable(const std::string& warehouse, const std::shared_ptr& schema,
+                     const std::map& options) const {
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
+        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                                       /*partition_keys=*/{}, /*primary_keys=*/{}, options,
+                                       /*ignore_if_exists=*/false));
+    }
+
+    std::unique_ptr MakeBatch(const std::shared_ptr& schema,
+                                           const std::string& json) const {
+        auto struct_type = arrow::struct_(schema->fields());
+        auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie();
+        ::ArrowArray arrow_array;
+        EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok());
+        RecordBatchBuilder batch_builder(&arrow_array);
+        return batch_builder.SetBucket(0).Finish().value();
+    }
+
+    std::vector> WriteAndPrepare(
+        const std::string& table_path, const std::shared_ptr& schema,
+        const std::map& options, const std::string& json,
+        int64_t commit_identifier) const {
+        WriteContextBuilder builder(table_path, commit_user_);
+        builder.SetOptions(options);
+        EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+        EXPECT_OK_AND_ASSIGN(auto file_store_write,
+                             FileStoreWrite::Create(std::move(write_context)));
+        EXPECT_OK(file_store_write->Write(MakeBatch(schema, json)));
+        EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                                   /*wait_compaction=*/false, commit_identifier));
+        EXPECT_OK(file_store_write->Close());
+        return commit_msgs;
+    }
+
+    std::vector> WriteAndPrepareWithWriteSchema(
+        const std::string& table_path, const std::shared_ptr& schema,
+        const std::map& options,
+        const std::vector& write_schema, const std::string& json,
+        int64_t commit_identifier) const {
+        WriteContextBuilder builder(table_path, commit_user_);
+        builder.SetOptions(options).WithWriteSchema(write_schema);
+        EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+        EXPECT_OK_AND_ASSIGN(auto file_store_write,
+                             FileStoreWrite::Create(std::move(write_context)));
+        EXPECT_OK(file_store_write->Write(MakeBatch(schema, json)));
+        EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                                   /*wait_compaction=*/false, commit_identifier));
+        EXPECT_OK(file_store_write->Close());
+        return commit_msgs;
+    }
+
+    void Commit(const std::string& table_path, const std::map& options,
+                const std::vector>& commit_msgs) const {
+        CommitContextBuilder builder(table_path, commit_user_);
+        builder.SetOptions(options);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish());
+        ASSERT_OK_AND_ASSIGN(auto file_store_commit,
+                             FileStoreCommit::Create(std::move(commit_context)));
+        ASSERT_OK(file_store_commit->Commit(commit_msgs));
+    }
+
+    std::shared_ptr OnlyNewFile(
+        const std::vector>& commit_msgs) const {
+        EXPECT_EQ(1, commit_msgs.size());
+        auto msg = std::dynamic_pointer_cast(commit_msgs[0]);
+        EXPECT_NE(nullptr, msg);
+        EXPECT_EQ(1, msg->GetNewFilesIncrement().NewFiles().size());
+        return msg->GetNewFilesIncrement().NewFiles()[0];
+    }
+
+    std::shared_ptr ReadDataFileSchema(
+        const std::string& table_path, const std::shared_ptr& file,
+        const std::map& options) const {
+        std::string file_path =
+            PathUtil::JoinPath(PathUtil::JoinPath(table_path, "bucket-0"), file->file_name);
+        auto fs = std::make_shared();
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path));
+        EXPECT_OK_AND_ASSIGN(auto format_str, file->FileFormat());
+        EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format_str, options));
+        EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(10));
+        EXPECT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream));
+        EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema());
+        return arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
+    }
+
+    MapSharedShreddingFieldMeta ShreddingMeta(const std::shared_ptr& file_schema,
+                                              int32_t field_index) const {
+        auto metadata = file_schema->field(field_index)->metadata();
+        EXPECT_NE(nullptr, metadata);
+        return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
+    }
+
  private:
     arrow::FieldVector fields_;
     std::string commit_user_;
@@ -174,4 +282,326 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition)
     }
 }
 
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFiles) {
+    std::map options = {
+        {"file.format", "parquet"},
+        {"target-file-row-num", "1"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "10"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+        {"bucket-key", "id"},
+    };
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, options);
+
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    WriteContextBuilder builder(table_path, commit_user_);
+    builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [1, [["a", 1], ["b", 2]]]
+    ])")));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [2, [["c", 3], ["d", 4], ["e", 5]]]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                               /*wait_compaction=*/false, /*commit_identifier=*/0));
+    ASSERT_OK(file_store_write->Close());
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
+    auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/1);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_first_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, first_meta.num_columns);
+    ASSERT_EQ(2, first_meta.max_row_width);
+
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
+    auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_second_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}}));
+    ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false));
+    ASSERT_EQ(2, second_meta.num_columns);
+    ASSERT_EQ(3, second_meta.max_row_width);
+}
+
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExistingAvroFile) {
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+
+    std::map avro_options = {
+        {"file.format", "avro"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+        {"bucket-key", "id"},
+    };
+    std::map shredding_options = avro_options;
+    shredding_options["file.format"] = "parquet";
+    shredding_options["fields.tags.map.storage-layout"] = "shared-shredding";
+    shredding_options["fields.tags.map.shared-shredding.max-columns"] = "10";
+    shredding_options["fields.tags.map.shared-shredding.column-placement-policy"] = "plain";
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, avro_options);
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto avro_commit_msgs = WriteAndPrepare(table_path, logical_schema, avro_options, R"([
+        [1, [["a", 1], ["b", 2]]]
+    ])",
+                                            /*commit_identifier=*/0);
+    Commit(table_path, avro_options, avro_commit_msgs);
+
+    auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([
+        [2, [["c", 3], ["d", 4], ["e", 5]]]
+    ])",
+                                              /*commit_identifier=*/1);
+    auto second_file_schema =
+        ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options);
+    auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1);
+
+    ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   logical_schema, {{"tags", 10}}));
+    ASSERT_TRUE(second_file_schema->Equals(*expected_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, second_meta.num_columns);
+    ASSERT_EQ(3, second_meta.max_row_width);
+}
+
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptAcrossRollingFiles) {
+    std::map options = {
+        {"file.format", "parquet"},
+        {"target-file-row-num", "1"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "10"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+        {"fields.attrs.map.storage-layout", "shared-shredding"},
+        {"fields.attrs.map.shared-shredding.max-columns", "10"},
+        {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+        {"bucket-key", "id"},
+    };
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+        arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, options);
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    WriteContextBuilder builder(table_path, commit_user_);
+    builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]]
+    ])")));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [2, [["g", 7], ["h", 8], ["i", 9]], [["j", 10], ["k", 11], ["l", 12], ["m", 13], ["n", 14]]]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                               /*wait_compaction=*/false, /*commit_identifier=*/0));
+    ASSERT_OK(file_store_write->Close());
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
+    auto first_tags_meta = ShreddingMeta(first_file_schema, /*field_index=*/1);
+    auto first_attrs_meta = ShreddingMeta(first_file_schema, /*field_index=*/2);
+    ASSERT_OK_AND_ASSIGN(auto expected_first_schema,
+                         MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                             logical_schema, {{"tags", 10}, {"attrs", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, first_tags_meta.num_columns);
+    ASSERT_EQ(2, first_tags_meta.max_row_width);
+    ASSERT_EQ(10, first_attrs_meta.num_columns);
+    ASSERT_EQ(4, first_attrs_meta.max_row_width);
+
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
+    auto second_tags_meta = ShreddingMeta(second_file_schema, /*field_index=*/1);
+    auto second_attrs_meta = ShreddingMeta(second_file_schema, /*field_index=*/2);
+    ASSERT_OK_AND_ASSIGN(auto expected_second_schema,
+                         MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                             logical_schema, {{"tags", 2}, {"attrs", 4}}));
+    ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false));
+    ASSERT_EQ(2, second_tags_meta.num_columns);
+    ASSERT_EQ(3, second_tags_meta.max_row_width);
+    ASSERT_EQ(4, second_attrs_meta.num_columns);
+    ASSERT_EQ(5, second_attrs_meta.max_row_width);
+}
+
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterUsesMaxForEveryMap) {
+    std::map options = {
+        {"file.format", "parquet"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "10"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+        {"fields.attrs.map.storage-layout", "shared-shredding"},
+        {"fields.attrs.map.shared-shredding.max-columns", "10"},
+        {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+        {"bucket-key", "id"},
+    };
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+        arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    auto tags_schema = arrow::schema({
+        logical_schema->field(0),
+        logical_schema->field(1),
+    });
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, options);
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto tags_commit_msgs =
+        WriteAndPrepareWithWriteSchema(table_path, tags_schema, options, {"id", "tags"}, R"([
+            [1, [["a", 1], ["b", 2]]]
+        ])",
+                                       /*commit_identifier=*/0);
+    Commit(table_path, options, tags_commit_msgs);
+
+    auto full_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+        [2, [["c", 3], ["d", 4], ["e", 5]], [["f", 6], ["g", 7], ["h", 8], ["i", 9]]]
+    ])",
+                                            /*commit_identifier=*/1);
+    auto full_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(full_commit_msgs), options);
+    auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1);
+    auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2);
+
+    ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   logical_schema, {{"tags", 10}, {"attrs", 10}}));
+    ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, tags_meta.num_columns);
+    ASSERT_EQ(3, tags_meta.max_row_width);
+    ASSERT_EQ(10, attrs_meta.num_columns);
+    ASSERT_EQ(4, attrs_meta.max_row_width);
+}
+
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingFieldDictCompressionFileRoundTrip) {
+    std::vector file_formats = {"parquet"};
+#ifdef PAIMON_ENABLE_ORC
+    file_formats.emplace_back("orc");
+#endif
+    for (const std::string& file_format : file_formats) {
+        for (const std::string compression : {"none", "zstd"}) {
+            SCOPED_TRACE("format=" + file_format + ", compression=" + compression);
+            std::map options = {
+                {Options::FILE_FORMAT, file_format},
+                {Options::FILE_COMPRESSION, compression},
+                {"fields.tags.map.storage-layout", "shared-shredding"},
+                {"fields.tags.map.shared-shredding.max-columns", "4"},
+                {"write-only", "true"},
+                {"bucket", "1"},
+                {"bucket-key", "id"},
+            };
+            auto logical_schema = arrow::schema({
+                arrow::field("id", arrow::int32()),
+                arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+            });
+
+            auto dir = UniqueTestDirectory::Create();
+            ASSERT_TRUE(dir);
+            CreateTable(dir->Str(), logical_schema, options);
+            std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+            auto commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+                [1, [["a", 1], ["b", 2]]],
+                [2, [["c", 3]]]
+            ])",
+                                               /*commit_identifier=*/0);
+
+            auto file_schema = ReadDataFileSchema(table_path, OnlyNewFile(commit_msgs), options);
+            auto tags_metadata = file_schema->GetFieldByName("tags")->metadata();
+            ASSERT_NE(tags_metadata, nullptr);
+            int32_t compression_index =
+                tags_metadata->FindKey(MapSharedShreddingDefine::kFieldDictCompression);
+            ASSERT_GE(compression_index, 0);
+            ASSERT_EQ(compression, tags_metadata->value(compression_index));
+
+            ASSERT_OK_AND_ASSIGN(
+                MapSharedShreddingFieldMeta field_meta,
+                MapSharedShreddingUtils::DeserializeMetadata(tags_metadata->Copy()));
+            ASSERT_EQ(3, field_meta.name_to_id.size());
+            ASSERT_TRUE(field_meta.name_to_id.count("a"));
+            ASSERT_TRUE(field_meta.name_to_id.count("b"));
+            ASSERT_TRUE(field_meta.name_to_id.count("c"));
+        }
+    }
+}
+
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingPartialWriteSkipsMissingMapColumn) {
+    std::map options = {
+        {"file.format", "parquet"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "10"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+        {"bucket-key", "id"},
+    };
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32()),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    auto id_only_schema = arrow::schema({
+        logical_schema->field(0),
+    });
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, options);
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+        [1, [["a", 1], ["b", 2]]]
+    ])",
+                                             /*commit_identifier=*/0);
+    Commit(table_path, options, first_commit_msgs);
+
+    auto id_only_commit_msgs =
+        WriteAndPrepareWithWriteSchema(table_path, id_only_schema, options, {"id"}, R"([
+            [2]
+        ])",
+                                       /*commit_identifier=*/1);
+    auto id_only_file_schema =
+        ReadDataFileSchema(table_path, OnlyNewFile(id_only_commit_msgs), options);
+
+    ASSERT_TRUE(id_only_file_schema->Equals(*id_only_schema, /*check_metadata=*/false));
+    for (const auto& field : id_only_file_schema->fields()) {
+        auto metadata = field->metadata() ? field->metadata()->Copy() : nullptr;
+        ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata));
+    }
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/blob_file_context.cpp b/src/paimon/core/operation/blob_file_context.cpp
index de6cd857..047cc6fb 100644
--- a/src/paimon/core/operation/blob_file_context.cpp
+++ b/src/paimon/core/operation/blob_file_context.cpp
@@ -30,98 +30,68 @@ namespace paimon {
 BlobFileContext::BlobFileContext(std::set descriptor_fields,
                                  std::set view_fields,
                                  std::set inline_fields,
-                                 std::set external_storage_fields,
-                                 std::set blob_file_fields,
-                                 std::optional external_storage_path)
+                                 std::set blob_file_fields)
     : descriptor_fields_(std::move(descriptor_fields)),
       view_fields_(std::move(view_fields)),
       inline_fields_(std::move(inline_fields)),
-      external_storage_fields_(std::move(external_storage_fields)),
-      blob_file_fields_(std::move(blob_file_fields)),
-      external_storage_path_(std::move(external_storage_path)) {}
+      blob_file_fields_(std::move(blob_file_fields)) {}
 
 std::unique_ptr BlobFileContext::Create(
     const std::shared_ptr& schema, const CoreOptions& options) {
-    // Check if there are any BLOB fields in the schema
-    bool has_blob = false;
+    // Collect the BLOB field names that are present in the given schema. The schema may
+    // only contain a subset of the table columns (e.g. a projected read/write schema), so all
+    // field categories below must be derived from this set rather than from the options
+    // alone, which describe the full table.
+    std::set schema_blob_fields;
     for (int i = 0; i < schema->num_fields(); ++i) {
-        if (BlobUtils::IsBlobField(schema->field(i))) {
-            has_blob = true;
-            break;
+        const auto& field = schema->field(i);
+        if (BlobUtils::IsBlobField(field)) {
+            schema_blob_fields.insert(field->name());
         }
     }
-    if (!has_blob) {
+    if (schema_blob_fields.empty()) {
         return nullptr;
     }
 
     // Populate descriptor fields
     std::set descriptor_fields;
     for (const auto& name : options.GetBlobDescriptorFields()) {
-        descriptor_fields.insert(name);
+        if (schema_blob_fields.count(name) > 0) {
+            descriptor_fields.insert(name);
+        }
     }
 
     // Populate view fields
     std::set view_fields;
     for (const auto& name : options.GetBlobViewFields()) {
-        view_fields.insert(name);
+        if (schema_blob_fields.count(name) > 0) {
+            view_fields.insert(name);
+        }
     }
 
     // Populate inline fields from options (descriptor ∪ view)
     std::set inline_fields;
     for (const auto& name : options.GetBlobInlineFields()) {
-        inline_fields.insert(name);
-    }
-
-    // Populate external storage fields
-    std::set external_storage_fields;
-    for (const auto& name : options.GetBlobExternalStorageFields()) {
-        external_storage_fields.insert(name);
+        if (schema_blob_fields.count(name) > 0) {
+            inline_fields.insert(name);
+        }
     }
 
-    // Populate external storage path
-    std::optional external_storage_path = options.GetBlobExternalStoragePath();
-
-    // Determine blob_file_fields: BLOB fields that are NOT inline
+    // Determine blob_file_fields: schema BLOB fields that are NOT inline
     std::set blob_file_fields;
-    for (int i = 0; i < schema->num_fields(); ++i) {
-        const auto& field = schema->field(i);
-        if (BlobUtils::IsBlobField(field) && inline_fields.count(field->name()) == 0) {
-            blob_file_fields.insert(field->name());
+    for (const auto& name : schema_blob_fields) {
+        if (inline_fields.count(name) == 0) {
+            blob_file_fields.insert(name);
         }
     }
 
     return std::unique_ptr(
         new BlobFileContext(std::move(descriptor_fields), std::move(view_fields),
-                            std::move(inline_fields), std::move(external_storage_fields),
-                            std::move(blob_file_fields), std::move(external_storage_path)));
-}
-
-bool BlobFileContext::IsInlineField(const std::string& field_name) const {
-    return inline_fields_.count(field_name) > 0;
-}
-
-bool BlobFileContext::IsBlobFileField(const std::string& field_name) const {
-    return blob_file_fields_.count(field_name) > 0;
-}
-
-bool BlobFileContext::IsDescriptorField(const std::string& field_name) const {
-    return descriptor_fields_.count(field_name) > 0;
-}
-
-bool BlobFileContext::IsViewField(const std::string& field_name) const {
-    return view_fields_.count(field_name) > 0;
-}
-
-bool BlobFileContext::IsExternalStorageField(const std::string& field_name) const {
-    return external_storage_fields_.count(field_name) > 0;
+                            std::move(inline_fields), std::move(blob_file_fields)));
 }
 
 bool BlobFileContext::RequireBlobFileWriter() const {
     return !blob_file_fields_.empty();
 }
 
-bool BlobFileContext::RequireExternalStorageWriter() const {
-    return !external_storage_fields_.empty();
-}
-
 }  // namespace paimon
diff --git a/src/paimon/core/operation/blob_file_context.h b/src/paimon/core/operation/blob_file_context.h
index 36f524ba..ee3b7019 100644
--- a/src/paimon/core/operation/blob_file_context.h
+++ b/src/paimon/core/operation/blob_file_context.h
@@ -20,7 +20,6 @@
 #pragma once
 
 #include 
-#include 
 #include 
 #include 
 
@@ -38,8 +37,6 @@ class CoreOptions;
 ///   - descriptor_fields: stored as BlobDescriptor bytes inline in the main data file.
 ///   - view_fields: stored as BlobViewStruct bytes inline in the main data file.
 ///   - inline_fields: descriptor_fields ∪ view_fields. These stay in the main data file.
-///   - external_storage_fields: subset of descriptor_fields whose raw data is written to an
-///     external storage path (the descriptor still goes into the main data file).
 ///   - blob_file_fields: BLOB fields that are NOT inline. These go into separate .blob files.
 class BlobFileContext {
  public:
@@ -49,28 +46,9 @@ class BlobFileContext {
     static std::unique_ptr Create(const std::shared_ptr& schema,
                                                    const CoreOptions& options);
 
-    /// Returns true if the given field should be stored inline in the main data file
-    /// (either as descriptor bytes or view bytes).
-    bool IsInlineField(const std::string& field_name) const;
-
-    /// Returns true if the given field should be written to a separate .blob file.
-    bool IsBlobFileField(const std::string& field_name) const;
-
-    /// Returns true if the given field is a descriptor field.
-    bool IsDescriptorField(const std::string& field_name) const;
-
-    /// Returns true if the given field is a view field.
-    bool IsViewField(const std::string& field_name) const;
-
-    /// Returns true if the given field should be written to external storage.
-    bool IsExternalStorageField(const std::string& field_name) const;
-
     /// Returns true if there are any BLOB fields that need a .blob file writer.
     bool RequireBlobFileWriter() const;
 
-    /// Returns true if there are any external storage fields that need an external writer.
-    bool RequireExternalStorageWriter() const;
-
     const std::set& GetDescriptorFields() const {
         return descriptor_fields_;
     }
@@ -83,31 +61,18 @@ class BlobFileContext {
         return inline_fields_;
     }
 
-    const std::set& GetExternalStorageFields() const {
-        return external_storage_fields_;
-    }
-
     const std::set& GetBlobFileFields() const {
         return blob_file_fields_;
     }
 
-    const std::optional& GetExternalStoragePath() const {
-        return external_storage_path_;
-    }
-
  private:
     BlobFileContext(std::set descriptor_fields, std::set view_fields,
-                    std::set inline_fields,
-                    std::set external_storage_fields,
-                    std::set blob_file_fields,
-                    std::optional external_storage_path);
+                    std::set inline_fields, std::set blob_file_fields);
 
     std::set descriptor_fields_;
     std::set view_fields_;
     std::set inline_fields_;
-    std::set external_storage_fields_;
     std::set blob_file_fields_;
-    std::optional external_storage_path_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/blob_file_context_test.cpp b/src/paimon/core/operation/blob_file_context_test.cpp
index a5d9f75e..a18dc333 100644
--- a/src/paimon/core/operation/blob_file_context_test.cpp
+++ b/src/paimon/core/operation/blob_file_context_test.cpp
@@ -69,7 +69,6 @@ TEST_F(BlobFileContextTest, AllInlineNoExternalStorage) {
     ASSERT_EQ(context->GetInlineFields(), std::set({"image", "video"}));
     ASSERT_TRUE(context->GetBlobFileFields().empty());
     ASSERT_FALSE(context->RequireBlobFileWriter());
-    ASSERT_FALSE(context->RequireExternalStorageWriter());
 }
 
 TEST_F(BlobFileContextTest, MixedInlineAndBlobFile) {
@@ -94,46 +93,8 @@ TEST_F(BlobFileContextTest, MixedInlineAndBlobFile) {
     // blob file fields = non-inline blob fields
     ASSERT_EQ(context->GetBlobFileFields(), std::set({"video", "audio"}));
 
-    // Query methods
-    ASSERT_TRUE(context->IsInlineField("image"));
-    ASSERT_TRUE(context->IsDescriptorField("image"));
-    ASSERT_FALSE(context->IsViewField("image"));
-    ASSERT_FALSE(context->IsBlobFileField("image"));
-
-    ASSERT_FALSE(context->IsInlineField("video"));
-    ASSERT_TRUE(context->IsBlobFileField("video"));
-
-    ASSERT_FALSE(context->IsInlineField("audio"));
-    ASSERT_TRUE(context->IsBlobFileField("audio"));
-
     // Requires blob file writer for video and audio
     ASSERT_TRUE(context->RequireBlobFileWriter());
-    ASSERT_FALSE(context->RequireExternalStorageWriter());
-}
-
-TEST_F(BlobFileContextTest, ExternalStorageFields) {
-    auto schema = MakeSchema({"id"}, {"image", "video"});
-    std::map opts_map = {
-        {Options::BLOB_DESCRIPTOR_FIELD, "image,video"},
-        {Options::BLOB_EXTERNAL_STORAGE_FIELD, "image"},
-        {Options::BLOB_EXTERNAL_STORAGE_PATH, "oss://bucket/blob/"},
-    };
-    ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map));
-    auto context = BlobFileContext::Create(schema, options);
-    ASSERT_TRUE(context);
-
-    ASSERT_EQ(context->GetDescriptorFields(), std::set({"image", "video"}));
-    ASSERT_EQ(context->GetInlineFields(), std::set({"image", "video"}));
-    ASSERT_EQ(context->GetExternalStorageFields(), std::set({"image"}));
-    ASSERT_TRUE(context->GetExternalStoragePath());
-    ASSERT_EQ(context->GetExternalStoragePath(), "oss://bucket/blob/");
-    ASSERT_TRUE(context->GetBlobFileFields().empty());
-
-    ASSERT_TRUE(context->IsExternalStorageField("image"));
-    ASSERT_FALSE(context->IsExternalStorageField("video"));
-
-    ASSERT_FALSE(context->RequireBlobFileWriter());
-    ASSERT_TRUE(context->RequireExternalStorageWriter());
 }
 
 TEST_F(BlobFileContextTest, ViewFields) {
@@ -151,12 +112,7 @@ TEST_F(BlobFileContextTest, ViewFields) {
     ASSERT_EQ(context->GetInlineFields(), std::set({"ref_image"}));
     ASSERT_EQ(context->GetBlobFileFields(), std::set({"raw_blob"}));
 
-    ASSERT_TRUE(context->IsInlineField("ref_image"));
-    ASSERT_TRUE(context->IsViewField("ref_image"));
-    ASSERT_FALSE(context->IsDescriptorField("ref_image"));
-
     ASSERT_TRUE(context->RequireBlobFileWriter());
-    ASSERT_FALSE(context->RequireExternalStorageWriter());
 }
 
 TEST_F(BlobFileContextTest, DescriptorAndViewTogether) {
@@ -164,8 +120,6 @@ TEST_F(BlobFileContextTest, DescriptorAndViewTogether) {
     std::map opts_map = {
         {Options::BLOB_DESCRIPTOR_FIELD, "desc_blob"},
         {Options::BLOB_VIEW_FIELD, "view_blob"},
-        {Options::BLOB_EXTERNAL_STORAGE_FIELD, "desc_blob"},
-        {Options::BLOB_EXTERNAL_STORAGE_PATH, "/tmp/ext/"},
     };
     ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map));
     auto context = BlobFileContext::Create(schema, options);
@@ -174,29 +128,49 @@ TEST_F(BlobFileContextTest, DescriptorAndViewTogether) {
     ASSERT_EQ(context->GetDescriptorFields(), std::set({"desc_blob"}));
     ASSERT_EQ(context->GetViewFields(), std::set({"view_blob"}));
     ASSERT_EQ(context->GetInlineFields(), std::set({"desc_blob", "view_blob"}));
-    ASSERT_EQ(context->GetExternalStorageFields(), std::set({"desc_blob"}));
-    ASSERT_TRUE(context->GetExternalStoragePath());
-    ASSERT_EQ(context->GetExternalStoragePath(), "/tmp/ext/");
     ASSERT_EQ(context->GetBlobFileFields(), std::set({"normal_blob"}));
 
-    ASSERT_TRUE(context->IsDescriptorField("desc_blob"));
-    ASSERT_TRUE(context->IsExternalStorageField("desc_blob"));
-    ASSERT_TRUE(context->IsInlineField("desc_blob"));
-    ASSERT_FALSE(context->IsBlobFileField("desc_blob"));
+    ASSERT_TRUE(context->RequireBlobFileWriter());
+}
+
+TEST_F(BlobFileContextTest, PartialSchemaIgnoresAbsentFields) {
+    // Schema only carries "image"; "video" and "audio" are not part of this write schema.
+    auto schema = MakeSchema({"id"}, {"image"});
+    std::map opts_map = {
+        {Options::BLOB_DESCRIPTOR_FIELD, "image,audio"},
+        {Options::BLOB_VIEW_FIELD, "video"},
+    };
+    ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map));
+    auto context = BlobFileContext::Create(schema, options);
+    ASSERT_TRUE(context);
 
-    ASSERT_TRUE(context->IsViewField("view_blob"));
-    ASSERT_TRUE(context->IsInlineField("view_blob"));
-    ASSERT_FALSE(context->IsDescriptorField("view_blob"));
+    // Only "image" survives filtering; "audio" / "video" are not in the schema.
+    ASSERT_EQ(context->GetDescriptorFields(), std::set({"image"}));
+    ASSERT_TRUE(context->GetViewFields().empty());
+    ASSERT_EQ(context->GetInlineFields(), std::set({"image"}));
 
-    ASSERT_FALSE(context->IsInlineField("normal_blob"));
-    ASSERT_TRUE(context->IsBlobFileField("normal_blob"));
+    // No non-inline blob field remains in the schema.
+    ASSERT_TRUE(context->GetBlobFileFields().empty());
 
-    // Non-existent field
-    ASSERT_FALSE(context->IsInlineField("not_exist"));
-    ASSERT_FALSE(context->IsBlobFileField("not_exist"));
+    ASSERT_FALSE(context->RequireBlobFileWriter());
+}
+
+TEST_F(BlobFileContextTest, PartialSchemaWithOnlyBlobFileField) {
+    auto schema = MakeSchema({"id"}, {"audio"});
+    std::map opts_map = {
+        {Options::BLOB_DESCRIPTOR_FIELD, "image"}, {Options::BLOB_VIEW_FIELD, "video"},
+        // "audio" is not configured as inline -> goes to .blob file
+    };
+    ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map));
+    auto context = BlobFileContext::Create(schema, options);
+    ASSERT_TRUE(context);
+
+    ASSERT_TRUE(context->GetDescriptorFields().empty());
+    ASSERT_TRUE(context->GetViewFields().empty());
+    ASSERT_TRUE(context->GetInlineFields().empty());
+    ASSERT_EQ(context->GetBlobFileFields(), std::set({"audio"}));
 
     ASSERT_TRUE(context->RequireBlobFileWriter());
-    ASSERT_TRUE(context->RequireExternalStorageWriter());
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/commit/commit_changes_provider.cpp b/src/paimon/core/operation/commit/commit_changes_provider.cpp
new file mode 100644
index 00000000..019b2e1d
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_changes_provider.cpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+
+#include 
+
+namespace paimon {
+
+namespace {
+
+class FixedInputCommitChangesProvider final : public CommitChangesProvider {
+ public:
+    FixedInputCommitChangesProvider(std::vector delta_files,
+                                    std::vector changelog_files,
+                                    std::vector index_entries)
+        : commit_changes_(std::make_shared(
+              std::move(delta_files), std::move(changelog_files), std::move(index_entries))) {}
+
+    Result> Provide(const std::optional&) const override {
+        return commit_changes_;
+    }
+
+ private:
+    std::shared_ptr commit_changes_;
+};
+
+}  // namespace
+
+std::shared_ptr CommitChangesProvider::Provider(
+    std::vector delta_files, std::vector changelog_files,
+    std::vector index_entries) {
+    return std::make_shared(
+        std::move(delta_files), std::move(changelog_files), std::move(index_entries));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/commit_changes_provider.h b/src/paimon/core/operation/commit/commit_changes_provider.h
new file mode 100644
index 00000000..ef23eb0d
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_changes_provider.h
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+struct CommitChanges {
+    CommitChanges() = default;
+
+    CommitChanges(std::vector delta, std::vector changelog,
+                  std::vector index)
+        : delta_files(std::move(delta)),
+          changelog_files(std::move(changelog)),
+          index_entries(std::move(index)) {}
+
+    std::vector delta_files;
+    std::vector changelog_files;
+    std::vector index_entries;
+};
+
+class CommitChangesProvider {
+ public:
+    virtual ~CommitChangesProvider() = default;
+
+    static std::shared_ptr Provider(
+        std::vector delta_files, std::vector changelog_files,
+        std::vector index_entries);
+
+    virtual Result> Provide(
+        const std::optional& latest_snapshot) const = 0;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp
new file mode 100644
index 00000000..edbb1a48
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+BinaryRow CreateIntRow(int32_t value) {
+    BinaryRow row(1);
+    BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+    writer.WriteInt(0, value);
+    writer.Complete();
+    return row;
+}
+
+ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind,
+                                  int32_t partition_value) {
+    auto file_meta = std::make_shared(
+        file_name, /*file_size=*/1024, /*row_count=*/8, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_seq_no=*/0,
+        /*max_seq_no=*/0,
+        /*schema_id=*/0, /*level=*/0,
+        /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(0, 0),
+        /*delete_row_count=*/std::nullopt,
+        /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+        /*value_stats_cols=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1,
+                         file_meta);
+}
+
+IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value) {
+    auto index_file = std::make_shared(
+        /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1,
+        /*dv_ranges=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*global_index_meta=*/std::nullopt);
+
+    return IndexManifestEntry(FileKind::Add(), CreateIntRow(partition_value), /*bucket=*/0,
+                              index_file);
+}
+
+}  // namespace
+
+TEST(CommitChangesProviderTest, TestProvideReturnsGivenEntries) {
+    std::vector delta_files = {
+        CreateManifestEntry("delta-1", FileKind::Add(), /*partition_value=*/1)};
+    std::vector changelog_files = {
+        CreateManifestEntry("changelog-1", FileKind::Add(), /*partition_value=*/2)};
+    std::vector index_entries = {
+        CreateIndexEntry("index-1", /*partition_value=*/3)};
+
+    std::shared_ptr provider =
+        CommitChangesProvider::Provider(delta_files, changelog_files, index_entries);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider->Provide(std::nullopt));
+
+    const std::vector& provided_delta = provided->delta_files;
+    const std::vector& provided_changelog = provided->changelog_files;
+    const std::vector& provided_index = provided->index_entries;
+
+    ASSERT_EQ(delta_files.size(), provided_delta.size());
+    ASSERT_EQ(changelog_files.size(), provided_changelog.size());
+    ASSERT_EQ(index_entries.size(), provided_index.size());
+    ASSERT_EQ("delta-1", provided_delta[0].FileName());
+    ASSERT_EQ("changelog-1", provided_changelog[0].FileName());
+    ASSERT_EQ("index-1", provided_index[0].index_file->FileName());
+}
+
+TEST(CommitChangesProviderTest, TestProvideUsesCopiedInputs) {
+    std::vector delta_files = {
+        CreateManifestEntry("delta-1", FileKind::Add(), /*partition_value=*/1)};
+    std::vector changelog_files;
+    std::vector index_entries;
+
+    std::shared_ptr provider =
+        CommitChangesProvider::Provider(delta_files, changelog_files, index_entries);
+
+    delta_files.push_back(CreateManifestEntry("delta-2", FileKind::Add(), /*partition_value=*/2));
+    changelog_files.push_back(
+        CreateManifestEntry("changelog-2", FileKind::Add(), /*partition_value=*/3));
+    index_entries.push_back(CreateIndexEntry("index-2", /*partition_value=*/4));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider->Provide(std::nullopt));
+
+    ASSERT_EQ(1u, provided->delta_files.size());
+    ASSERT_EQ(0u, provided->changelog_files.size());
+    ASSERT_EQ(0u, provided->index_entries.size());
+    ASSERT_EQ("delta-1", provided->delta_files[0].FileName());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/commit_scanner.cpp b/src/paimon/core/operation/commit/commit_scanner.cpp
new file mode 100644
index 00000000..abee7695
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_scanner.cpp
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/commit_scanner.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/operation/commit/overwrite_changes_provider.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/table/bucket_mode.h"
+#include "paimon/scan_context.h"
+
+namespace paimon {
+
+CommitScanner::CommitScanner(const std::shared_ptr& snapshot_manager,
+                             const std::shared_ptr& schema_manager,
+                             const std::shared_ptr& manifest_list,
+                             const std::shared_ptr& manifest_file,
+                             const std::shared_ptr& index_manifest_file,
+                             const std::shared_ptr& table_schema,
+                             const std::shared_ptr& schema,
+                             const CoreOptions& core_options,
+                             const std::shared_ptr& executor,
+                             const std::shared_ptr& pool,
+                             const BinaryRowPartitionComputer* partition_computer,
+                             ScanSupplier scan_supplier)
+    : snapshot_manager_(snapshot_manager),
+      schema_manager_(schema_manager),
+      manifest_list_(manifest_list),
+      manifest_file_(manifest_file),
+      index_manifest_file_(index_manifest_file),
+      table_schema_(table_schema),
+      schema_(schema),
+      core_options_(core_options),
+      executor_(executor),
+      pool_(pool),
+      partition_computer_(partition_computer),
+      scan_supplier_(std::move(scan_supplier)) {}
+
+Result>> CommitScanner::ToPartitionFilters(
+    const std::vector& changed_partitions) const {
+    std::vector> partition_filters;
+    partition_filters.reserve(changed_partitions.size());
+
+    for (const BinaryRow& changed_partition : changed_partitions) {
+        std::vector> part_values;
+        PAIMON_ASSIGN_OR_RAISE(part_values,
+                               partition_computer_->GeneratePartitionVector(changed_partition));
+        std::map partition_filter;
+        for (const auto& [key, value] : part_values) {
+            partition_filter[key] = value;
+        }
+        partition_filters.push_back(std::move(partition_filter));
+    }
+
+    return partition_filters;
+}
+
+Result> CommitScanner::ReadAllEntriesFromChangedPartitions(
+    const Snapshot& snapshot, const std::vector& changed_partitions) const {
+    if (changed_partitions.empty()) {
+        return std::vector{};
+    }
+
+    std::vector> partition_filters;
+    PAIMON_ASSIGN_OR_RAISE(partition_filters, ToPartitionFilters(changed_partitions));
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan,
+                           NewScan(partition_filters, /*for_overwrite=*/false));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan,
+                           scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan());
+    return plan->Files();
+}
+
+Result> CommitScanner::ReadIncrementalEntries(
+    const Snapshot& snapshot, const std::vector& changed_partitions) const {
+    if (changed_partitions.empty()) {
+        return std::vector{};
+    }
+
+    std::unordered_set changed_partition_set(changed_partitions.begin(),
+                                                        changed_partitions.end());
+    std::vector delta_manifests;
+    PAIMON_RETURN_NOT_OK(manifest_list_->ReadDeltaManifests(snapshot, &delta_manifests));
+
+    std::vector incremental_entries;
+    for (const ManifestFileMeta& manifest_meta : delta_manifests) {
+        std::vector manifest_entries;
+        PAIMON_RETURN_NOT_OK(
+            manifest_file_->Read(manifest_meta.FileName(), /*filter=*/nullptr, &manifest_entries));
+        for (const ManifestEntry& entry : manifest_entries) {
+            if (changed_partition_set.find(entry.Partition()) != changed_partition_set.end()) {
+                incremental_entries.push_back(entry);
+            }
+        }
+    }
+
+    return incremental_entries;
+}
+
+Result> CommitScanner::ReadAllEntriesFromPartitions(
+    const Snapshot& snapshot,
+    const std::vector>& partitions) const {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan,
+                           NewScan(partitions, /*for_overwrite=*/false));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan,
+                           scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan());
+    return plan->Files();
+}
+
+Result> CommitScanner::NewScan(
+    const std::vector>& partitions, bool for_overwrite) const {
+    auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions,
+                                                    /*bucket_filter=*/std::nullopt);
+    if (!scan_supplier_) {
+        return Status::Invalid("CommitScanner requires non-empty scan supplier.");
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, scan_supplier_(scan_filter));
+    if (for_overwrite && core_options_.GetBucket() != BucketModeDefine::POSTPONE_BUCKET) {
+        scan->OnlyReadRealBuckets();
+    }
+    return scan;
+}
+
+Result> CommitScanner::ReadAllIndexEntriesFromPartitions(
+    const Snapshot& snapshot,
+    const std::vector>& partitions) const {
+    std::vector index_entries;
+    if (!snapshot.IndexManifest()) {
+        return index_entries;
+    }
+
+    auto filter = [this, &partitions](const IndexManifestEntry& entry) -> Result {
+        if (partitions.empty()) {
+            return true;
+        }
+
+        std::vector> part_values;
+        PAIMON_ASSIGN_OR_RAISE(part_values,
+                               partition_computer_->GeneratePartitionVector(entry.partition));
+        std::map partition;
+        for (const auto& [key, value] : part_values) {
+            partition[key] = value;
+        }
+
+        for (const auto& partition_spec : partitions) {
+            bool matched = true;
+            for (const auto& [key, value] : partition_spec) {
+                auto iter = partition.find(key);
+                if (iter == partition.end() || iter->second != value) {
+                    matched = false;
+                    break;
+                }
+            }
+            if (matched) {
+                return true;
+            }
+        }
+        return false;
+    };
+
+    PAIMON_RETURN_NOT_OK(
+        index_manifest_file_->Read(snapshot.IndexManifest().value(), filter, &index_entries));
+    return index_entries;
+}
+
+std::shared_ptr CommitScanner::OverwriteChangesProvider(
+    const std::vector>& partitions,
+    const std::vector& changes,
+    const std::vector& index_entries) const {
+    return std::make_shared(
+        changes, index_entries,
+        [this, partitions](const Snapshot& snapshot) -> Result> {
+            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan,
+                                   NewScan(partitions, /*for_overwrite=*/true));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr plan,
+                scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan());
+            return plan->Files();
+        },
+        [this, partitions](const Snapshot& snapshot) {
+            return ReadAllIndexEntriesFromPartitions(snapshot, partitions);
+        });
+}
+
+Result> CommitScanner::ReadTotalBuckets(
+    const Snapshot& snapshot, const std::vector& changed_partitions) const {
+    std::unordered_map total_buckets;
+    if (changed_partitions.empty()) {
+        return total_buckets;
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector entries,
+                           ReadAllEntriesFromChangedPartitions(snapshot, changed_partitions));
+
+    std::unordered_set remaining_partitions(changed_partitions.begin(),
+                                                       changed_partitions.end());
+    for (const ManifestEntry& entry : entries) {
+        if (remaining_partitions.empty()) {
+            break;
+        }
+        if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0) {
+            continue;
+        }
+        if (remaining_partitions.erase(entry.Partition()) > 0) {
+            total_buckets.emplace(entry.Partition(), entry.TotalBuckets());
+        }
+    }
+
+    return total_buckets;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/commit_scanner.h b/src/paimon/core/operation/commit/commit_scanner.h
new file mode 100644
index 00000000..552cc648
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_scanner.h
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/core_options.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class BinaryRowPartitionComputer;
+class CommitChangesProvider;
+class Executor;
+class FileStoreScan;
+struct IndexManifestEntry;
+class IndexManifestFile;
+class ManifestEntry;
+class ManifestFile;
+class ManifestList;
+class MemoryPool;
+class ScanFilter;
+class SchemaManager;
+class Snapshot;
+class SnapshotManager;
+class TableSchema;
+
+/// Manifest entries scanner for commit operations.
+class CommitScanner {
+ public:
+    using ScanSupplier =
+        std::function>(const std::shared_ptr&)>;
+
+    CommitScanner(const std::shared_ptr& snapshot_manager,
+                  const std::shared_ptr& schema_manager,
+                  const std::shared_ptr& manifest_list,
+                  const std::shared_ptr& manifest_file,
+                  const std::shared_ptr& index_manifest_file,
+                  const std::shared_ptr& table_schema,
+                  const std::shared_ptr& schema, const CoreOptions& core_options,
+                  const std::shared_ptr& executor,
+                  const std::shared_ptr& pool,
+                  const BinaryRowPartitionComputer* partition_computer, ScanSupplier scan_supplier);
+
+    Result> ReadAllEntriesFromChangedPartitions(
+        const Snapshot& snapshot, const std::vector& changed_partitions) const;
+
+    Result> ReadIncrementalEntries(
+        const Snapshot& snapshot, const std::vector& changed_partitions) const;
+
+    Result> ReadTotalBuckets(
+        const Snapshot& snapshot, const std::vector& changed_partitions) const;
+
+    Result> ReadAllEntriesFromPartitions(
+        const Snapshot& snapshot,
+        const std::vector>& partitions) const;
+
+    Result> ReadAllIndexEntriesFromPartitions(
+        const Snapshot& snapshot,
+        const std::vector>& partitions) const;
+
+    std::shared_ptr OverwriteChangesProvider(
+        const std::vector>& partitions,
+        const std::vector& changes,
+        const std::vector& index_entries) const;
+
+ private:
+    Result>> ToPartitionFilters(
+        const std::vector& changed_partitions) const;
+
+    Result> NewScan(
+        const std::vector>& partitions,
+        bool for_overwrite) const;
+
+ private:
+    std::shared_ptr snapshot_manager_;
+    std::shared_ptr schema_manager_;
+    std::shared_ptr manifest_list_;
+    std::shared_ptr manifest_file_;
+    std::shared_ptr index_manifest_file_;
+    std::shared_ptr table_schema_;
+    std::shared_ptr schema_;
+    CoreOptions core_options_;
+    std::shared_ptr executor_;
+    std::shared_ptr pool_;
+    const BinaryRowPartitionComputer* partition_computer_;
+    ScanSupplier scan_supplier_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp
new file mode 100644
index 00000000..3d5c6270
--- /dev/null
+++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/commit_scanner.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/scan_context.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+Snapshot MakeSnapshot(std::optional index_manifest = std::nullopt) {
+    return Snapshot(
+        /*id=*/1,
+        /*schema_id=*/0,
+        /*base_manifest_list=*/"base-manifest-list",
+        /*base_manifest_list_size=*/std::nullopt,
+        /*delta_manifest_list=*/"delta-manifest-list",
+        /*delta_manifest_list_size=*/std::nullopt,
+        /*changelog_manifest_list=*/std::nullopt,
+        /*changelog_manifest_list_size=*/std::nullopt,
+        /*index_manifest=*/std::move(index_manifest),
+        /*commit_user=*/"test-user",
+        /*commit_identifier=*/1, Snapshot::CommitKind::Append(),
+        /*time_millis=*/0,
+        /*total_record_count=*/0,
+        /*delta_record_count=*/0,
+        /*changelog_record_count=*/std::nullopt,
+        /*watermark=*/std::nullopt,
+        /*statistics=*/std::nullopt,
+        /*properties=*/std::nullopt,
+        /*next_row_id=*/std::nullopt);
+}
+
+BinaryRow CreateIntPartition(int32_t value) {
+    BinaryRow row(1);
+    BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+    writer.WriteInt(0, value);
+    writer.Complete();
+    return row;
+}
+
+IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value) {
+    auto index_file = std::make_shared(
+        /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1,
+        /*dv_ranges=*/std::nullopt,
+        /*external_path=*/std::nullopt);
+    return IndexManifestEntry(FileKind::Add(), CreateIntPartition(partition_value),
+                              /*bucket=*/0, index_file);
+}
+
+}  // namespace
+
+class CommitScannerTest : public testing::Test {
+ protected:
+    void SetUp() override {
+        dir_ = UniqueTestDirectory::Create();
+        ASSERT_TRUE(dir_);
+        schema_ = arrow::schema({arrow::field("pt", arrow::int32())});
+        ASSERT_OK_AND_ASSIGN(core_options_, CoreOptions::FromMap({}));
+        ASSERT_OK_AND_ASSIGN(partition_computer_,
+                             BinaryRowPartitionComputer::Create(
+                                 /*partition_keys=*/{"pt"}, schema_,
+                                 /*default_part_value=*/"__DEFAULT_PARTITION__",
+                                 /*legacy_partition_name_enabled=*/true, GetDefaultPool()));
+    }
+
+    CommitScanner CreateScanner(
+        CommitScanner::ScanSupplier scan_supplier,
+        const std::shared_ptr& index_manifest_file = nullptr) const {
+        return CommitScanner(
+            /*snapshot_manager=*/nullptr,
+            /*schema_manager=*/nullptr,
+            /*manifest_list=*/nullptr,
+            /*manifest_file=*/nullptr,
+            /*index_manifest_file=*/index_manifest_file,
+            /*table_schema=*/nullptr, schema_, core_options_,
+            /*executor=*/nullptr, GetDefaultPool(), partition_computer_.get(),
+            std::move(scan_supplier));
+    }
+
+    Result> CreateIndexManifestFile() const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_format,
+                               FileFormatFactory::Get("orc", {}));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr path_factory,
+            FileStorePathFactory::Create(
+                dir_->Str(), schema_, /*partition_keys=*/{"pt"},
+                /*default_part_value=*/"__DEFAULT_PARTITION__", file_format->Identifier(),
+                /*data_file_prefix=*/"data-",
+                /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                /*global_index_external_path=*/std::nullopt,
+                /*index_file_in_data_file_dir=*/false, GetDefaultPool()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr index_manifest_file,
+            IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory,
+                                      /*bucket_mode=*/2, GetDefaultPool(), core_options_));
+        return std::shared_ptr(std::move(index_manifest_file));
+    }
+
+ protected:
+    std::unique_ptr dir_;
+    std::shared_ptr schema_;
+    CoreOptions core_options_;
+    std::unique_ptr partition_computer_;
+};
+
+TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsEmptyFastExit) {
+    bool supplier_called = false;
+    CommitScanner scanner = CreateScanner([&supplier_called](const std::shared_ptr&)
+                                              -> Result> {
+        supplier_called = true;
+        return Status::Invalid("should not be called");
+    });
+
+    ASSERT_OK_AND_ASSIGN(std::vector entries,
+                         scanner.ReadAllEntriesFromChangedPartitions(MakeSnapshot(),
+                                                                     /*changed_partitions=*/{}));
+    ASSERT_TRUE(entries.empty());
+    ASSERT_FALSE(supplier_called);
+}
+
+TEST_F(CommitScannerTest, TestReadAllEntriesFromPartitionsRequiresSupplier) {
+    CommitScanner scanner = CreateScanner(CommitScanner::ScanSupplier{});
+
+    std::vector> partitions = {{{"pt", "1"}}};
+    ASSERT_NOK_WITH_MSG(scanner.ReadAllEntriesFromPartitions(MakeSnapshot(), partitions),
+                        "CommitScanner requires non-empty scan supplier");
+}
+
+TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsBuildsScanFilterPartitions) {
+    std::vector> captured_partition_filters;
+    bool supplier_called = false;
+
+    CommitScanner scanner = CreateScanner(
+        [&captured_partition_filters, &supplier_called](
+            const std::shared_ptr& filter) -> Result> {
+            supplier_called = true;
+            captured_partition_filters = filter->GetPartitionFilters();
+            return Status::Invalid("stop after capturing filter");
+        });
+
+    std::vector changed_partitions = {CreateIntPartition(42)};
+    ASSERT_NOK_WITH_MSG(
+        scanner.ReadAllEntriesFromChangedPartitions(MakeSnapshot(), changed_partitions),
+        "stop after capturing filter");
+
+    ASSERT_TRUE(supplier_called);
+    ASSERT_EQ(1u, captured_partition_filters.size());
+    ASSERT_EQ(1u, captured_partition_filters[0].size());
+    ASSERT_EQ("42", captured_partition_filters[0]["pt"]);
+}
+
+TEST_F(CommitScannerTest, TestReadAllIndexEntriesFromPartitions) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr index_manifest_file,
+                         CreateIndexManifestFile());
+    std::vector entries = {CreateIndexEntry("index-1", /*partition_value=*/1),
+                                               CreateIndexEntry("index-2", /*partition_value=*/2),
+                                               CreateIndexEntry("index-3", /*partition_value=*/3)};
+    ASSERT_OK_AND_ASSIGN(
+        std::optional index_manifest,
+        index_manifest_file->WriteIndexFiles(/*previous_index_manifest=*/std::nullopt, entries));
+    ASSERT_TRUE(index_manifest);
+
+    CommitScanner scanner = CreateScanner(CommitScanner::ScanSupplier{}, index_manifest_file);
+    Snapshot snapshot = MakeSnapshot(index_manifest);
+
+    ASSERT_OK_AND_ASSIGN(std::vector unfiltered,
+                         scanner.ReadAllIndexEntriesFromPartitions(snapshot, /*partitions=*/{}));
+    ASSERT_EQ(3u, unfiltered.size());
+
+    std::vector> partitions = {
+        {{"pt", "2"}}, {{"unknown_partition_key", "value"}}};
+    ASSERT_OK_AND_ASSIGN(std::vector filtered,
+                         scanner.ReadAllIndexEntriesFromPartitions(snapshot, partitions));
+    ASSERT_EQ(1u, filtered.size());
+    ASSERT_EQ("index-2", filtered[0].index_file->FileName());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp
new file mode 100644
index 00000000..4e5b4289
--- /dev/null
+++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h"
+
+#include 
+#include 
+
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+
+namespace paimon {
+
+bool CompactedChangelogPathResolver::IsCompactedChangelogPath(const std::string& path) {
+    const std::string file_name = PathUtil::GetName(path);
+    return StringUtils::StartsWith(file_name, "compacted-changelog-");
+}
+
+std::string CompactedChangelogPathResolver::Resolve(const std::string& path) {
+    if (!IsCompactedChangelogPath(path)) {
+        return path;
+    }
+    const std::string file_name = PathUtil::GetName(path);
+
+    const size_t dot_pos = file_name.find_last_of('.');
+    if (dot_pos == std::string::npos || dot_pos + 1 >= file_name.size()) {
+        return path;
+    }
+
+    const std::string name_without_ext = file_name.substr(0, dot_pos);
+    const std::string format = file_name.substr(dot_pos + 1);
+    const size_t dollar_pos = name_without_ext.find('$');
+    if (dollar_pos == std::string::npos || dollar_pos + 1 >= name_without_ext.size()) {
+        return path;
+    }
+
+    const std::string base_name = name_without_ext.substr(0, dollar_pos);
+    const std::string suffix = name_without_ext.substr(dollar_pos + 1);
+    const std::vector split_tokens = StringUtils::Split(suffix, "-", false);
+
+    // Real compacted changelog path pattern: ...$bucket-len.ext
+    if (split_tokens.size() == 2) {
+        return path;
+    }
+
+    // Fake compacted changelog path pattern: ...$bucket-len-offset-sliceLen.ext
+    if (split_tokens.size() < 4) {
+        return path;
+    }
+
+    const std::string& bucket = split_tokens[0];
+    const std::string& total_len = split_tokens[1];
+    const std::string real_file_name = base_name + "$" + bucket + "-" + total_len + "." + format;
+
+    const std::string parent = PathUtil::GetParentDirPath(path);
+    const std::string grand_parent = PathUtil::GetParentDirPath(parent);
+    if (parent.empty() || grand_parent.empty()) {
+        return path;
+    }
+
+    return PathUtil::JoinPath(PathUtil::JoinPath(grand_parent, "bucket-" + bucket), real_file_name);
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h
new file mode 100644
index 00000000..427e8f35
--- /dev/null
+++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+
+namespace paimon {
+
+/// Utility class for resolving compacted changelog file paths.
+///
+/// This class provides functionality to resolve fake compacted changelog file paths to their real
+/// file paths.
+///
+/// File Name Protocol
+///
+/// There are two kinds of file name. In the following description, `bid1` and `bid2` are bucket
+/// id, `off` is offset, `len1` and `len2` are lengths.
+///
+/// - `bucket-bid1/compacted-changelog-xxx$bid1-len1`: This is the real file name. If this file
+///   name is recorded in manifest file meta, reader should read the bytes of this file starting
+///   from offset `0` with length `len1`.
+/// - `bucket-bid2/compacted-changelog-xxx$bid1-len1-off-len2`: This is the fake file name. Reader
+///   should read the bytes of file `bucket-bid1/compacted-changelog-xxx$bid1-len1` starting from
+///   offset `off` with length `len2`.
+class CompactedChangelogPathResolver {
+ public:
+    /// Resolves a file path, handling compacted changelog file path resolution if applicable.
+    ///
+    /// For compacted changelog files, resolves fake file paths to their real file paths as
+    /// described in the protocol above. For non-compacted changelog files, returns the path
+    /// unchanged.
+    ///
+    /// @param path The file path to resolve.
+    /// @return The resolved real file path for compacted changelog files, or the original path
+    ///     unchanged for other files.
+    static std::string Resolve(const std::string& path);
+
+ private:
+    /// Checks if the given path is a compacted changelog file path.
+    ///
+    /// @param path The file path to check.
+    /// @return true if the path is a compacted changelog file, false otherwise.
+    static bool IsCompactedChangelogPath(const std::string& path);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp b/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp
new file mode 100644
index 00000000..ef9bfc5a
--- /dev/null
+++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h"
+
+#include 
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+
+TEST(CompactedChangelogPathResolverTest, IsCompactedChangelogPath) {
+    std::string regular_changelog =
+        "table/bucket-0/changelog-25b05ab0-6f90-4865-a984-8d9629bac735-1426.parquet";
+    std::string compacted_changelog =
+        "table/bucket-0/"
+        "compacted-changelog-8e049c65-5ce4-4ce7-b1b0-78ce694ab351$0-39253.cc-parquet";
+    std::string data_file = "table/bucket-0/data-file-1.parquet";
+
+    ASSERT_FALSE(CompactedChangelogPathResolver::IsCompactedChangelogPath(regular_changelog));
+    ASSERT_TRUE(CompactedChangelogPathResolver::IsCompactedChangelogPath(compacted_changelog));
+    ASSERT_FALSE(CompactedChangelogPathResolver::IsCompactedChangelogPath(data_file));
+}
+
+TEST(CompactedChangelogPathResolverTest, ResolveNonCompactedChangelogPath) {
+    std::string regular_changelog =
+        "table/bucket-0/changelog-25b05ab0-6f90-4865-a984-8d9629bac735-1426.parquet";
+
+    ASSERT_EQ(regular_changelog, CompactedChangelogPathResolver::Resolve(regular_changelog));
+}
+
+TEST(CompactedChangelogPathResolverTest, ResolveFakeCompactedChangelogPath) {
+    std::string fake_path =
+        "table/f1=10/bucket-1/compacted-changelog-abc$0-39253-39253-35699.cc-parquet";
+    std::string expected_real_path =
+        "table/f1=10/bucket-0/compacted-changelog-abc$0-39253.cc-parquet";
+
+    ASSERT_EQ(expected_real_path, CompactedChangelogPathResolver::Resolve(fake_path));
+}
+
+TEST(CompactedChangelogPathResolverTest, KeepRealCompactedChangelogPath) {
+    std::string real_path = "table/f1=10/bucket-1/compacted-changelog-abc$1-39253.cc-parquet";
+
+    ASSERT_EQ(real_path, CompactedChangelogPathResolver::Resolve(real_path));
+}
+
+TEST(CompactedChangelogPathResolverTest, KeepNonCompactedPath) {
+    std::string normal_path = "table/f1=10/bucket-1/data-file.orc";
+
+    ASSERT_EQ(normal_path, CompactedChangelogPathResolver::Resolve(normal_path));
+}
+
+TEST(CompactedChangelogPathResolverTest, KeepInvalidCompactedPath) {
+    std::string invalid_path = "table/f1=10/bucket-1/compacted-changelog-abc$1.cc-parquet";
+
+    ASSERT_EQ(invalid_path, CompactedChangelogPathResolver::Resolve(invalid_path));
+}
+
+TEST(CompactedChangelogPathResolverTest, ResolveWithDifferentFormats) {
+    std::string fake_orc_path =
+        "table/f1=10/bucket-2/compacted-changelog-abc$0-1024-1024-512.cc-orc";
+    std::string expected_orc_path = "table/f1=10/bucket-0/compacted-changelog-abc$0-1024.cc-orc";
+    ASSERT_EQ(expected_orc_path, CompactedChangelogPathResolver::Resolve(fake_orc_path));
+
+    std::string fake_avro_path =
+        "table/f1=10/bucket-5/compacted-changelog-abc$2-2048-2048-1024.cc-avro";
+    std::string expected_avro_path = "table/f1=10/bucket-2/compacted-changelog-abc$2-2048.cc-avro";
+    ASSERT_EQ(expected_avro_path, CompactedChangelogPathResolver::Resolve(fake_avro_path));
+}
+
+TEST(CompactedChangelogPathResolverTest, ResolveFileWithoutExtension) {
+    std::string file_without_extension = "table/f1=10/bucket-0/file";
+
+    ASSERT_EQ(file_without_extension,
+              CompactedChangelogPathResolver::Resolve(file_without_extension));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/conflict_detection.cpp b/src/paimon/core/operation/commit/conflict_detection.cpp
new file mode 100644
index 00000000..1d62fc47
--- /dev/null
+++ b/src/paimon/core/operation/commit/conflict_detection.cpp
@@ -0,0 +1,759 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/conflict_detection.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/range_helper.h"
+#include "paimon/common/utils/vector_store_utils.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/manifest/file_entry.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_file.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/operation/commit/commit_scanner.h"
+#include "paimon/core/operation/commit/manifest_entry_changes.h"
+#include "paimon/core/operation/commit/row_id_column_conflict_checker.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/utils/range.h"
+#include "paimon/utils/row_range_index.h"
+
+namespace paimon {
+
+namespace {
+
+bool IsDedicatedStorageFile(const std::string& file_name) {
+    return BlobUtils::IsBlobFile(file_name) || VectorStoreUtils::IsVectorStoreFile(file_name);
+}
+
+struct PartitionBucketKey {
+    BinaryRow partition;
+    int32_t bucket;
+
+    bool operator==(const PartitionBucketKey& other) const {
+        return partition == other.partition && bucket == other.bucket;
+    }
+};
+
+struct PartitionBucketKeyHash {
+    size_t operator()(const PartitionBucketKey& key) const {
+        return std::hash()(key.partition) ^ (std::hash()(key.bucket) << 1);
+    }
+};
+
+}  // namespace
+
+ConflictDetection::ConflictDetection(std::shared_ptr table_schema,
+                                     const CoreOptions& options,
+                                     std::shared_ptr snapshot_manager,
+                                     std::shared_ptr manifest_list,
+                                     std::shared_ptr manifest_file,
+                                     std::shared_ptr commit_scanner,
+                                     const std::string& commit_user, const std::string& table_name,
+                                     const std::shared_ptr& path_factory)
+    : table_schema_(std::move(table_schema)),
+      options_(options),
+      snapshot_manager_(std::move(snapshot_manager)),
+      manifest_list_(std::move(manifest_list)),
+      manifest_file_(std::move(manifest_file)),
+      commit_scanner_(std::move(commit_scanner)),
+      path_factory_(path_factory),
+      commit_user_(commit_user),
+      table_name_(table_name) {}
+
+void ConflictDetection::SetRowIdCheckFromSnapshot(
+    const std::optional& row_id_check_from_snapshot) {
+    row_id_check_from_snapshot_ = row_id_check_from_snapshot;
+}
+
+bool ConflictDetection::HasRowIdCheckFromSnapshot() const {
+    return row_id_check_from_snapshot_.has_value();
+}
+
+Status ConflictDetection::CheckConflicts(
+    const Snapshot& latest_snapshot, const std::vector& base_entries,
+    const std::vector& delta_entries,
+    const std::vector& delta_index_entries,
+    const std::optional>&
+        row_id_column_conflict_checker,
+    const Snapshot::CommitKind& commit_kind) const {
+    std::string base_commit_user = latest_snapshot.CommitUser();
+    if (options_.DeletionVectorsEnabled() &&
+        ResolveBucketMode(options_.GetBucket(), table_schema_) == BucketMode::BUCKET_UNAWARE) {
+        return Status::NotImplemented(
+            "check conflicts failed. not yet support dv with BUCKET_UNAWARE mode");
+    }
+
+    std::vector all_entries = base_entries;
+    all_entries.insert(all_entries.end(), delta_entries.begin(), delta_entries.end());
+    PAIMON_RETURN_NOT_OK(CheckBucketKeepSame(all_entries, commit_kind, base_commit_user,
+                                             base_entries, delta_entries));
+
+    // check the delta, it is important not to delete and add the same file. Since scan
+    // relies on map for deduplication, this may result in the loss of this file
+    std::vector merged_delta_entries;
+    if (Status status = FileEntry::MergeEntries(delta_entries, &merged_delta_entries);
+        !status.ok()) {
+        return Status::Invalid(
+            BuildConflictMessage("File deletion conflicts detected! Give up committing.",
+                                 base_commit_user, base_entries, delta_entries, status.ToString()));
+    }
+
+    std::vector merged_entries;
+    // merge manifest entries and also check if the files we want to delete are still there
+    if (Status status = FileEntry::MergeEntries(all_entries, &merged_entries); !status.ok()) {
+        return Status::Invalid(
+            BuildConflictMessage("File deletion conflicts detected! Give up committing.",
+                                 base_commit_user, base_entries, delta_entries, status.ToString()));
+    }
+    PAIMON_RETURN_NOT_OK(
+        CheckDeleteInEntries(merged_entries, base_commit_user, base_entries, delta_entries));
+    PAIMON_RETURN_NOT_OK(
+        CheckKeyRange(merged_entries, base_commit_user, base_entries, delta_entries));
+    if (commit_kind != Snapshot::CommitKind::Compact()) {
+        PAIMON_RETURN_NOT_OK(
+            CheckRowIdExistence(base_entries, delta_entries, latest_snapshot.NextRowId()));
+    }
+    PAIMON_RETURN_NOT_OK(CheckRowIdRangeConflicts(commit_kind, merged_entries));
+    PAIMON_RETURN_NOT_OK(CheckGlobalIndexRowIdExistence(base_entries, delta_index_entries));
+    PAIMON_RETURN_NOT_OK(CheckForRowIdFromSnapshot(
+        latest_snapshot, delta_entries, delta_index_entries, row_id_column_conflict_checker));
+    return Status::OK();
+}
+
+bool ConflictDetection::ShouldBeOverwriteCommit(
+    const std::vector& append_table_files,
+    const std::vector& append_index_files) const {
+    for (const ManifestEntry& entry : append_table_files) {
+        if (entry.Kind() == FileKind::Delete()) {
+            return true;
+        }
+    }
+
+    for (const IndexManifestEntry& entry : append_index_files) {
+        if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+Status ConflictDetection::CheckBucketKeepSame(
+    const std::vector& all_entries, const Snapshot::CommitKind& commit_kind,
+    const std::string& base_commit_user, const std::vector& base_entries,
+    const std::vector& delta_entries) const {
+    if (commit_kind == Snapshot::CommitKind::Overwrite()) {
+        return Status::OK();
+    }
+
+    // total buckets within the same partition should remain the same
+    std::unordered_map total_buckets;
+    for (const ManifestEntry& entry : all_entries) {
+        if (entry.TotalBuckets() <= 0) {
+            continue;
+        }
+        if (same_bucket_checked_partitions_.find(entry.Partition()) !=
+            same_bucket_checked_partitions_.end()) {
+            continue;
+        }
+
+        auto [iter, inserted] = total_buckets.emplace(entry.Partition(), entry.TotalBuckets());
+        if (inserted || iter->second == entry.TotalBuckets()) {
+            continue;
+        }
+
+        return TotalBucketsChanged(entry.Partition(), entry.TotalBuckets(), iter->second,
+                                   base_commit_user, base_entries, delta_entries);
+    }
+
+    MarkBucketCheckedPartitions(total_buckets);
+    return Status::OK();
+}
+
+Status ConflictDetection::CollectUncheckedBucketPartitions(
+    const std::vector& delta_entries,
+    std::unordered_map* total_buckets) const {
+    total_buckets->clear();
+    for (const ManifestEntry& entry : delta_entries) {
+        if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0 ||
+            same_bucket_checked_partitions_.find(entry.Partition()) !=
+                same_bucket_checked_partitions_.end()) {
+            continue;
+        }
+
+        auto [iter, inserted] = total_buckets->emplace(entry.Partition(), entry.TotalBuckets());
+        if (!inserted && iter->second != entry.TotalBuckets()) {
+            return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second);
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckSameBucketByTotalBuckets(
+    const std::unordered_map& expected_total_buckets,
+    const std::unordered_map& previous_total_buckets) const {
+    for (const auto& [partition, total_buckets] : expected_total_buckets) {
+        auto iter = previous_total_buckets.find(partition);
+        if (iter != previous_total_buckets.end() && iter->second != total_buckets) {
+            return BucketNumMismatch(partition, total_buckets, iter->second);
+        }
+    }
+
+    MarkBucketCheckedPartitions(expected_total_buckets);
+    return Status::OK();
+}
+
+Status ConflictDetection::BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets,
+                                            int32_t previous_num_buckets) const {
+    std::string part_info;
+    if (table_schema_->PartitionKeys().empty()) {
+        part_info = "table";
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(std::string partition_string,
+                               path_factory_->GetPartitionString(partition));
+        part_info = fmt::format("partition {{{}}}", partition_string);
+    }
+    return Status::Invalid(fmt::format(
+        "Try to write {} with a new bucket num {}, but the previous bucket num is {}. Please "
+        "switch to batch mode, and perform INSERT OVERWRITE to rescale current data layout first.",
+        part_info, num_buckets, previous_num_buckets));
+}
+
+Status ConflictDetection::TotalBucketsChanged(
+    const BinaryRow& partition, int32_t num_buckets, int32_t previous_num_buckets,
+    const std::string& base_commit_user, const std::vector& base_entries,
+    const std::vector& delta_entries) const {
+    std::shared_ptr arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, table_schema_->PartitionKeys()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::string partition_string,
+        BinaryRowPartitionComputer::PartToSimpleString(partition_schema, partition, "-", 200,
+                                                       /*legacy_partition_name_enabled=*/false));
+    std::string message = fmt::format(
+        "Total buckets of partition {} changed from {} to {} without overwrite. Give up "
+        "committing.",
+        partition_string, previous_num_buckets, num_buckets);
+    return Status::Invalid(
+        BuildConflictMessage(message, base_commit_user, base_entries, delta_entries));
+}
+
+std::string ConflictDetection::BuildConflictMessage(const std::string& message,
+                                                    const std::string& base_commit_user,
+                                                    const std::vector& base_entries,
+                                                    const std::vector& delta_entries,
+                                                    const std::string& cause) const {
+    static constexpr const char* kPossibleCauses =
+        "Don't panic!\n"
+        "Conflicts during commits are normal and this failure is intended to resolve the "
+        "conflicts.\n"
+        "Conflicts are mainly caused by the following scenarios:\n"
+        "1. Multiple jobs are writing into the same partition at the same time, or you use "
+        "STATEMENT SET to execute multiple INSERT statements into the same Paimon table.\n"
+        "   You'll probably see different base commit user and current commit user below.\n"
+        "   You can use dedicated compaction job to support multiple writing.\n"
+        "2. You're recovering from an old savepoint, or you're creating multiple jobs from a "
+        "savepoint.\n"
+        "   The job will fail continuously in this scenario to protect metadata from corruption.\n"
+        "   You can either recover from the latest savepoint, or you can revert the table to the "
+        "snapshot corresponding to the old savepoint.";
+
+    constexpr size_t kMaxEntry = 50;
+    auto join_entries = [](const std::vector& entries,
+                           size_t max_entry) -> std::string {
+        std::string joined;
+        size_t limit = std::min(entries.size(), max_entry);
+        for (size_t i = 0; i < limit; ++i) {
+            if (i > 0) {
+                joined += "\n";
+            }
+            joined += entries[i].ToString();
+        }
+        return joined;
+    };
+
+    std::string commit_user_string = fmt::format(
+        "Base commit user is: {}; Current commit user is: {}", base_commit_user, commit_user_);
+    std::string base_entries_string = "Base entries are:\n" + join_entries(base_entries, kMaxEntry);
+    std::string changes_string = "Changes are:\n" + join_entries(delta_entries, kMaxEntry);
+
+    std::string result = fmt::format("{}\n\n{}\n\n{}\n\n{}\n\n{}", message, kPossibleCauses,
+                                     commit_user_string, base_entries_string, changes_string);
+    if (base_entries.size() > kMaxEntry || delta_entries.size() > kMaxEntry) {
+        result +=
+            "\n\nThe entry list above are not fully displayed, please refer to logs for more "
+            "information.";
+    }
+    if (!cause.empty()) {
+        result += "\n\nCaused by: " + cause;
+    }
+    return result;
+}
+
+void ConflictDetection::MarkBucketCheckedPartitions(
+    const std::unordered_map& total_buckets) const {
+    if (total_buckets.empty()) {
+        return;
+    }
+
+    for (const auto& [partition, _] : total_buckets) {
+        same_bucket_checked_partitions_.insert_or_assign(partition, true);
+        while (same_bucket_checked_partitions_.size() > kSameBucketCheckCacheMaxSize) {
+            same_bucket_checked_partitions_.erase(same_bucket_checked_partitions_.begin()->first);
+        }
+    }
+}
+
+Status ConflictDetection::CheckDeleteInEntries(
+    const std::vector& merged_entries, const std::string& base_commit_user,
+    const std::vector& base_entries,
+    const std::vector& delta_entries) const {
+    for (const auto& entry : merged_entries) {
+        if (entry.Kind() == FileKind::Delete()) {
+            std::string message = fmt::format(
+                "File deletion conflicts detected! Give up committing. Trying to delete file {} "
+                "for table {} which is not previously added.",
+                entry.FileName(), table_name_);
+            return Status::Invalid(
+                BuildConflictMessage(message, base_commit_user, base_entries, delta_entries));
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckKeyRange(const std::vector& merged_entries,
+                                        const std::string& base_commit_user,
+                                        const std::vector& base_entries,
+                                        const std::vector& delta_entries) const {
+    if (table_schema_->PrimaryKeys().empty()) {
+        return Status::OK();
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields,
+                           table_schema_->TrimmedPrimaryKeyFields());
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator,
+                           FieldsComparator::Create(trimmed_primary_key_fields,
+                                                    options_.SequenceFieldSortOrderIsAscending()));
+
+    // group entries by partitions, buckets and levels
+    std::unordered_map, std::vector> levels;
+    for (const auto& entry : merged_entries) {
+        if (!(entry.Kind() == FileKind::Add())) {
+            continue;
+        }
+        int32_t level = entry.Level();
+        if (level < 1) {
+            continue;
+        }
+
+        levels[std::make_tuple(entry.Partition(), entry.Bucket(), level)].push_back(entry);
+    }
+
+    // check for all LSM level >= 1, key ranges of files do not intersect
+    for (auto& [_, entries] : levels) {
+        std::sort(entries.begin(), entries.end(),
+                  [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) {
+                      return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0;
+                  });
+        for (size_t i = 0; i + 1 < entries.size(); ++i) {
+            const ManifestEntry& a = entries[i];
+            const ManifestEntry& b = entries[i + 1];
+            if (key_comparator->CompareTo(a.MaxKey(), b.MinKey()) >= 0) {
+                PAIMON_ASSIGN_OR_RAISE(std::string a_partition_string,
+                                       path_factory_->GetPartitionString(a.Partition()));
+                PAIMON_ASSIGN_OR_RAISE(std::string b_partition_string,
+                                       path_factory_->GetPartitionString(b.Partition()));
+                std::string message = fmt::format(
+                    "LSM conflicts detected! Give up committing. Conflict files are:\n"
+                    "{}, bucket {}, level {}, file {}\n"
+                    "{}, bucket {}, level {}, file {}",
+                    a_partition_string, a.Bucket(), a.Level(), a.FileName(), b_partition_string,
+                    b.Bucket(), b.Level(), b.FileName());
+                return Status::Invalid(
+                    BuildConflictMessage(message, base_commit_user, base_entries, delta_entries));
+            }
+        }
+    }
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckRowIdExistence(const std::vector& base_entries,
+                                              const std::vector& delta_entries,
+                                              const std::optional& next_row_id) const {
+    if (!options_.DataEvolutionEnabled()) {
+        return Status::OK();
+    }
+
+    std::vector files_to_check;
+    files_to_check.reserve(delta_entries.size());
+    for (const ManifestEntry& entry : delta_entries) {
+        if (!(entry.Kind() == FileKind::Add()) || !entry.File()->first_row_id || !next_row_id ||
+            entry.File()->first_row_id.value() >= next_row_id.value()) {
+            continue;
+        }
+        files_to_check.push_back(entry);
+    }
+    if (files_to_check.empty()) {
+        return Status::OK();
+    }
+
+    std::vector existing_data_ranges;
+    existing_data_ranges.reserve(base_entries.size());
+    for (const ManifestEntry& entry : base_entries) {
+        if (!entry.File()->first_row_id || IsDedicatedStorageFile(entry.FileName())) {
+            continue;
+        }
+        int64_t range_from = entry.File()->first_row_id.value();
+        int64_t range_to = range_from + entry.File()->row_count - 1;
+        existing_data_ranges.emplace_back(range_from, range_to);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(RowRangeIndex existing_index,
+                           RowRangeIndex::Create(existing_data_ranges,
+                                                 /*merge_adjacent=*/false));
+
+    for (const ManifestEntry& entry : files_to_check) {
+        int64_t range_from = entry.File()->first_row_id.value();
+        int64_t range_to = range_from + entry.File()->row_count - 1;
+        Range row_range(range_from, range_to);
+
+        bool exists = false;
+        if (IsDedicatedStorageFile(entry.FileName())) {
+            exists = existing_index.Contains(row_range);
+        } else {
+            exists = existing_index.ContainsExactly(row_range);
+        }
+
+        if (!exists) {
+            return Status::Invalid(fmt::format(
+                "Row ID existence conflict: file '{}' references firstRowId={}, rowCount={} in "
+                "bucket {}, but no matching file exists in the current snapshot. The referenced "
+                "file may have been rewritten by a concurrent compaction or removed by an "
+                "overwrite.",
+                entry.FileName(), entry.File()->first_row_id.value(), entry.File()->row_count,
+                entry.Bucket()));
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckRowIdRangeConflicts(
+    const Snapshot::CommitKind& commit_kind,
+    const std::vector& merged_entries) const {
+    if (!options_.DataEvolutionEnabled()) {
+        return Status::OK();
+    }
+    if (!row_id_check_from_snapshot_ && !(commit_kind == Snapshot::CommitKind::Compact())) {
+        return Status::OK();
+    }
+
+    std::vector entries_with_ranges;
+    entries_with_ranges.reserve(merged_entries.size());
+    for (const ManifestEntry& entry : merged_entries) {
+        if (entry.File()->first_row_id) {
+            entries_with_ranges.push_back(entry);
+        }
+    }
+    if (entries_with_ranges.empty()) {
+        return Status::OK();
+    }
+
+    RangeHelper range_helper(
+        [](const ManifestEntry& entry) -> Result {
+            return entry.File()->first_row_id.value();
+        },
+        [](const ManifestEntry& entry) -> Result {
+            return entry.File()->first_row_id.value() + entry.File()->row_count - 1;
+        });
+    std::vector data_files;
+    std::vector dedicated_files;
+    data_files.reserve(entries_with_ranges.size());
+    dedicated_files.reserve(entries_with_ranges.size());
+    for (const ManifestEntry& entry : entries_with_ranges) {
+        if (IsDedicatedStorageFile(entry.FileName())) {
+            dedicated_files.push_back(entry);
+        } else {
+            data_files.push_back(entry);
+        }
+    }
+
+    PAIMON_RETURN_NOT_OK(CheckDataFileRowIdRangeConflicts(range_helper, data_files));
+    PAIMON_RETURN_NOT_OK(CheckDedicatedFileRowIdRangeConflicts(data_files, dedicated_files));
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckDataFileRowIdRangeConflicts(
+    RangeHelper& range_helper, const std::vector& data_files) const {
+    std::vector data_files_copy = data_files;
+    PAIMON_ASSIGN_OR_RAISE(std::vector> data_file_groups,
+                           range_helper.MergeOverlappingRanges(std::move(data_files_copy)));
+    for (const std::vector& data_file_group : data_file_groups) {
+        PAIMON_ASSIGN_OR_RAISE(bool all_data_ranges_same,
+                               range_helper.AreAllRangesSame(data_file_group));
+        if (!all_data_ranges_same) {
+            std::string data_files_str;
+            for (size_t i = 0; i < data_file_group.size(); ++i) {
+                if (i > 0) {
+                    data_files_str += ", ";
+                }
+                data_files_str += data_file_group[i].ToString();
+            }
+            return Status::Invalid(fmt::format(
+                "For Data Evolution table, multiple 'MERGE INTO' and 'COMPACT' operations have "
+                "encountered conflicts, data files: [{}]",
+                data_files_str));
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckDedicatedFileRowIdRangeConflicts(
+    const std::vector& data_files,
+    const std::vector& dedicated_files) const {
+    if (dedicated_files.empty()) {
+        return Status::OK();
+    }
+
+    std::vector data_ranges;
+    data_ranges.reserve(data_files.size());
+    for (const ManifestEntry& data_file : data_files) {
+        int64_t data_range_from = data_file.File()->first_row_id.value();
+        int64_t data_range_to = data_range_from + data_file.File()->row_count - 1;
+        data_ranges.emplace_back(data_range_from, data_range_to);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(RowRangeIndex data_file_row_range_index,
+                           RowRangeIndex::Create(data_ranges, /*merge_adjacent=*/false));
+
+    for (const ManifestEntry& dedicated_file : dedicated_files) {
+        int64_t dedicated_from = dedicated_file.File()->first_row_id.value();
+        int64_t dedicated_to = dedicated_from + dedicated_file.File()->row_count - 1;
+        Range dedicated_range(dedicated_from, dedicated_to);
+
+        std::vector intersecting_ranges =
+            data_file_row_range_index.IntersectedRanges(dedicated_range.from, dedicated_range.to);
+        bool covered_by_one_data_range = intersecting_ranges.size() == 1 &&
+                                         intersecting_ranges[0].from <= dedicated_range.from &&
+                                         intersecting_ranges[0].to >= dedicated_range.to;
+        if (!covered_by_one_data_range) {
+            std::string conflict_reason = intersecting_ranges.size() > 1
+                                              ? "spans multiple data file ranges"
+                                              : "is not covered by one data file range";
+            std::string intersecting_files_str;
+            bool first = true;
+            for (const ManifestEntry& data_file : data_files) {
+                int64_t data_from = data_file.File()->first_row_id.value();
+                int64_t data_to = data_from + data_file.File()->row_count - 1;
+                if (data_from <= dedicated_range.to && dedicated_range.from <= data_to) {
+                    if (!first) {
+                        intersecting_files_str += ", ";
+                    }
+                    intersecting_files_str += data_file.ToString();
+                    first = false;
+                }
+            }
+            return Status::Invalid(fmt::format(
+                "For Data Evolution table, multiple 'MERGE INTO' and 'COMPACT' operations have "
+                "encountered conflicts, dedicated file {} {} {}: [{}]",
+                dedicated_file.ToString(), dedicated_range.ToString(), conflict_reason,
+                intersecting_files_str));
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckForRowIdFromSnapshot(
+    const Snapshot& latest_snapshot, const std::vector& delta_entries,
+    const std::vector& delta_index_entries,
+    const std::optional>&
+        row_id_column_conflict_checker) const {
+    if (!options_.DataEvolutionEnabled() || !row_id_check_from_snapshot_ || !snapshot_manager_ ||
+        !row_id_column_conflict_checker || !row_id_column_conflict_checker.value() ||
+        row_id_column_conflict_checker.value()->IsEmpty()) {
+        return Status::OK();
+    }
+
+    if (row_id_check_from_snapshot_.value() > latest_snapshot.Id()) {
+        return Status::OK();
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(Snapshot check_snapshot,
+                           snapshot_manager_->LoadSnapshot(row_id_check_from_snapshot_.value()));
+    if (!check_snapshot.NextRowId()) {
+        return Status::Invalid(fmt::format("Next row id cannot be null for snapshot {}.",
+                                           row_id_check_from_snapshot_.value()));
+    }
+    int64_t check_next_row_id = check_snapshot.NextRowId().value();
+
+    int64_t from_snapshot_id = row_id_check_from_snapshot_.value() + 1;
+    if (from_snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) {
+        from_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID;
+    }
+    if (from_snapshot_id > latest_snapshot.Id()) {
+        return Status::OK();
+    }
+
+    std::vector changed_partitions =
+        ManifestEntryChanges::ChangedPartitions(delta_entries, delta_index_entries);
+    if (changed_partitions.empty()) {
+        return Status::OK();
+    }
+    std::unordered_set changed_partition_set(changed_partitions.begin(),
+                                                        changed_partitions.end());
+
+    for (int64_t snapshot_id = from_snapshot_id; snapshot_id <= latest_snapshot.Id();
+         ++snapshot_id) {
+        PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id));
+        if (snapshot.GetCommitKind() == Snapshot::CommitKind::Compact()) {
+            continue;
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(
+            std::vector history_entries,
+            commit_scanner_->ReadIncrementalEntries(snapshot, changed_partitions));
+        for (const ManifestEntry& history_entry : history_entries) {
+            if (!(history_entry.Kind() == FileKind::Add()) || !history_entry.File()->first_row_id ||
+                changed_partition_set.find(history_entry.Partition()) ==
+                    changed_partition_set.end()) {
+                continue;
+            }
+            int64_t history_first_row_id = history_entry.File()->first_row_id.value();
+            if (history_first_row_id >= check_next_row_id) {
+                continue;
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                bool conflicts,
+                row_id_column_conflict_checker.value()->ConflictsWith(history_entry.File()));
+            if (conflicts) {
+                return Status::Invalid(
+                    "For Data Evolution table, multiple 'MERGE INTO' operations have "
+                    "encountered conflicts, updating the same file, which can render some "
+                    "updates ineffective.");
+            }
+        }
+    }
+
+    return Status::OK();
+}
+
+Status ConflictDetection::CheckGlobalIndexRowIdExistence(
+    const std::vector& base_entries,
+    const std::vector& delta_index_entries) const {
+    if (!options_.DataEvolutionEnabled()) {
+        return Status::OK();
+    }
+
+    std::vector indexes_to_check;
+    for (const IndexManifestEntry& index_entry : delta_index_entries) {
+        if (!(index_entry.kind == FileKind::Add()) ||
+            !index_entry.index_file->GetGlobalIndexMeta()) {
+            continue;
+        }
+        indexes_to_check.push_back(index_entry);
+    }
+    if (indexes_to_check.empty()) {
+        return Status::OK();
+    }
+
+    std::unordered_map, PartitionBucketKeyHash>
+        data_ranges_by_group;
+    for (const ManifestEntry& base_entry : base_entries) {
+        if (!(base_entry.Kind() == FileKind::Add()) || !base_entry.File()->first_row_id) {
+            continue;
+        }
+
+        int64_t first_row_id = base_entry.File()->first_row_id.value();
+        int64_t last_row_id = first_row_id + base_entry.File()->row_count - 1;
+        data_ranges_by_group[{base_entry.Partition(), base_entry.Bucket()}].emplace_back(
+            first_row_id, last_row_id);
+    }
+
+    std::unordered_map
+        range_index_by_group;
+    range_index_by_group.reserve(data_ranges_by_group.size());
+    for (const auto& [group, data_ranges] : data_ranges_by_group) {
+        PAIMON_ASSIGN_OR_RAISE(RowRangeIndex row_range_index,
+                               RowRangeIndex::Create(data_ranges, /*merge_adjacent=*/true));
+        range_index_by_group.emplace(group, std::move(row_range_index));
+    }
+
+    for (const IndexManifestEntry& index_entry : indexes_to_check) {
+        PartitionBucketKey group_key{index_entry.partition, index_entry.bucket};
+        auto group_iter = range_index_by_group.find(group_key);
+        if (group_iter == range_index_by_group.end()) {
+            return Status::Invalid(fmt::format(
+                "Global index row ID existence conflict: index file '{}' references row range {}, "
+                "but this range is not fully covered by current data files. The referenced row "
+                "IDs may have been reassigned or removed by a concurrent commit.",
+                index_entry.index_file->FileName(),
+                Range(index_entry.index_file->GetGlobalIndexMeta().value().row_range_start,
+                      index_entry.index_file->GetGlobalIndexMeta().value().row_range_end)
+                    .ToString()));
+        }
+
+        const GlobalIndexMeta& global_index = index_entry.index_file->GetGlobalIndexMeta().value();
+        Range index_range(global_index.row_range_start, global_index.row_range_end);
+
+        std::vector intersected =
+            group_iter->second.IntersectedRanges(index_range.from, index_range.to);
+        bool covered = intersected.size() == 1 && intersected[0].from <= index_range.from &&
+                       intersected[0].to >= index_range.to;
+        if (!covered) {
+            return Status::Invalid(fmt::format(
+                "Global index row ID existence conflict: index file '{}' references row range {}, "
+                "but this range is not fully covered by current data files. The referenced row "
+                "IDs may have been reassigned or removed by a concurrent commit.",
+                index_entry.index_file->FileName(), index_range.ToString()));
+        }
+    }
+
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/conflict_detection.h b/src/paimon/core/operation/commit/conflict_detection.h
new file mode 100644
index 00000000..de2793e2
--- /dev/null
+++ b/src/paimon/core/operation/commit/conflict_detection.h
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/common/utils/range_helper.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+class ManifestEntry;
+struct IndexManifestEntry;
+class CommitScanner;
+class FileStorePathFactory;
+class ManifestFile;
+class ManifestList;
+class RowIdColumnConflictChecker;
+class SnapshotManager;
+class TableSchema;
+
+/// Util class for detecting conflicts between base and delta files.
+class ConflictDetection {
+ public:
+    ConflictDetection(std::shared_ptr table_schema, const CoreOptions& options,
+                      std::shared_ptr snapshot_manager,
+                      std::shared_ptr manifest_list,
+                      std::shared_ptr manifest_file,
+                      std::shared_ptr commit_scanner, const std::string& commit_user,
+                      const std::string& table_name,
+                      const std::shared_ptr& path_factory);
+
+    Status CheckConflicts(const Snapshot& latest_snapshot,
+                          const std::vector& base_entries,
+                          const std::vector& delta_entries,
+                          const std::vector& delta_index_entries,
+                          const std::optional>&
+                              row_id_column_conflict_checker,
+                          const Snapshot::CommitKind& commit_kind) const;
+
+    void SetRowIdCheckFromSnapshot(const std::optional& row_id_check_from_snapshot);
+
+    bool HasRowIdCheckFromSnapshot() const;
+
+    bool ShouldBeOverwriteCommit(const std::vector& append_table_files,
+                                 const std::vector& append_index_files) const;
+
+    Status CollectUncheckedBucketPartitions(
+        const std::vector& delta_entries,
+        std::unordered_map* total_buckets) const;
+
+    Status CheckSameBucketByTotalBuckets(
+        const std::unordered_map& expected_total_buckets,
+        const std::unordered_map& previous_total_buckets) const;
+
+ private:
+    Status CheckBucketKeepSame(const std::vector& all_entries,
+                               const Snapshot::CommitKind& commit_kind,
+                               const std::string& base_commit_user,
+                               const std::vector& base_entries,
+                               const std::vector& delta_entries) const;
+
+    Status BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets,
+                             int32_t previous_num_buckets) const;
+
+    Status TotalBucketsChanged(const BinaryRow& partition, int32_t num_buckets,
+                               int32_t previous_num_buckets, const std::string& base_commit_user,
+                               const std::vector& base_entries,
+                               const std::vector& delta_entries) const;
+
+    std::string BuildConflictMessage(const std::string& message,
+                                     const std::string& base_commit_user,
+                                     const std::vector& base_entries,
+                                     const std::vector& delta_entries,
+                                     const std::string& cause = "") const;
+
+    void MarkBucketCheckedPartitions(
+        const std::unordered_map& total_buckets) const;
+
+    Status CheckDeleteInEntries(const std::vector& merged_entries,
+                                const std::string& base_commit_user,
+                                const std::vector& base_entries,
+                                const std::vector& delta_entries) const;
+
+    Status CheckKeyRange(const std::vector& merged_entries,
+                         const std::string& base_commit_user,
+                         const std::vector& base_entries,
+                         const std::vector& delta_entries) const;
+
+    Status CheckRowIdExistence(const std::vector& base_entries,
+                               const std::vector& delta_entries,
+                               const std::optional& next_row_id) const;
+
+    Status CheckRowIdRangeConflicts(const Snapshot::CommitKind& commit_kind,
+                                    const std::vector& merged_entries) const;
+
+    Status CheckDataFileRowIdRangeConflicts(RangeHelper& range_helper,
+                                            const std::vector& data_files) const;
+
+    Status CheckDedicatedFileRowIdRangeConflicts(
+        const std::vector& data_files,
+        const std::vector& dedicated_files) const;
+
+    Status CheckForRowIdFromSnapshot(
+        const Snapshot& latest_snapshot, const std::vector& delta_entries,
+        const std::vector& delta_index_entries,
+        const std::optional>&
+            row_id_column_conflict_checker) const;
+
+    Status CheckGlobalIndexRowIdExistence(
+        const std::vector& base_entries,
+        const std::vector& delta_index_entries) const;
+
+ private:
+    static constexpr size_t kSameBucketCheckCacheMaxSize = 1000;
+
+    std::shared_ptr table_schema_;
+    CoreOptions options_;
+    std::optional row_id_check_from_snapshot_;
+    std::shared_ptr snapshot_manager_;
+    std::shared_ptr manifest_list_;
+    std::shared_ptr manifest_file_;
+    std::shared_ptr commit_scanner_;
+    std::shared_ptr path_factory_;
+    std::string commit_user_;
+    std::string table_name_;
+    mutable LinkedHashMap same_bucket_checked_partitions_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp
new file mode 100644
index 00000000..28e4fed1
--- /dev/null
+++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp
@@ -0,0 +1,1049 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/conflict_detection.h"
+
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "arrow/type.h"
+#include "gtest/gtest.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/global_index_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+Snapshot MakeSnapshot(const Snapshot::CommitKind& commit_kind) {
+    return Snapshot(
+        /*id=*/1,
+        /*schema_id=*/1,
+        /*base_manifest_list=*/"base-manifest-list",
+        /*base_manifest_list_size=*/std::nullopt,
+        /*delta_manifest_list=*/"delta-manifest-list",
+        /*delta_manifest_list_size=*/std::nullopt,
+        /*changelog_manifest_list=*/std::nullopt,
+        /*changelog_manifest_list_size=*/std::nullopt,
+        /*index_manifest=*/std::nullopt,
+        /*commit_user=*/"test-user",
+        /*commit_identifier=*/1, commit_kind,
+        /*time_millis=*/0,
+        /*total_record_count=*/0,
+        /*delta_record_count=*/0,
+        /*changelog_record_count=*/std::nullopt,
+        /*watermark=*/std::nullopt,
+        /*statistics=*/std::nullopt,
+        /*properties=*/std::nullopt,
+        /*next_row_id=*/std::nullopt);
+}
+
+Snapshot MakeSnapshotWithNextRowId(const Snapshot::CommitKind& commit_kind,
+                                   const std::optional& next_row_id) {
+    return Snapshot(
+        /*id=*/1,
+        /*schema_id=*/1,
+        /*base_manifest_list=*/"base-manifest-list",
+        /*base_manifest_list_size=*/std::nullopt,
+        /*delta_manifest_list=*/"delta-manifest-list",
+        /*delta_manifest_list_size=*/std::nullopt,
+        /*changelog_manifest_list=*/std::nullopt,
+        /*changelog_manifest_list_size=*/std::nullopt,
+        /*index_manifest=*/std::nullopt,
+        /*commit_user=*/"test-user",
+        /*commit_identifier=*/1, commit_kind,
+        /*time_millis=*/0,
+        /*total_record_count=*/0,
+        /*delta_record_count=*/0,
+        /*changelog_record_count=*/std::nullopt,
+        /*watermark=*/std::nullopt,
+        /*statistics=*/std::nullopt,
+        /*properties=*/std::nullopt, next_row_id);
+}
+
+Status CheckConflicts(const ConflictDetection& detection,
+                      const std::vector& base_entries,
+                      const std::vector& delta_entries,
+                      const Snapshot::CommitKind& commit_kind) {
+    return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries,
+                                    /*delta_index_entries=*/{},
+                                    /*row_id_column_conflict_checker=*/std::nullopt, commit_kind);
+}
+
+Status CheckConflicts(const ConflictDetection& detection,
+                      const std::vector& base_entries,
+                      const std::vector& delta_entries,
+                      const std::vector& delta_index_entries,
+                      const Snapshot::CommitKind& commit_kind) {
+    return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries,
+                                    delta_index_entries,
+                                    /*row_id_column_conflict_checker=*/std::nullopt, commit_kind);
+}
+
+Status CheckConflictsWithNextRowId(const ConflictDetection& detection,
+                                   const std::vector& base_entries,
+                                   const std::vector& delta_entries,
+                                   const std::optional& next_row_id,
+                                   const Snapshot::CommitKind& commit_kind) {
+    return detection.CheckConflicts(MakeSnapshotWithNextRowId(commit_kind, next_row_id),
+                                    base_entries, delta_entries,
+                                    /*delta_index_entries=*/{},
+                                    /*row_id_column_conflict_checker=*/std::nullopt, commit_kind);
+}
+
+}  // namespace
+
+class ConflictDetectionTest : public testing::Test {
+ public:
+    void SetUp() override {
+        fields_ = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()),
+                   arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())};
+    }
+
+ protected:
+    std::shared_ptr CreatePathFactory(
+        const std::vector& partition_keys) const {
+        EXPECT_OK_AND_ASSIGN(
+            std::shared_ptr path_factory,
+            FileStorePathFactory::Create(
+                /*root=*/"/tmp/conflict_detection_test", arrow::schema(fields_), partition_keys,
+                /*default_part_value=*/"__DEFAULT_PARTITION__", /*identifier=*/"orc",
+                /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/false,
+                /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt,
+                /*index_file_in_data_file_dir=*/false, GetDefaultPool()));
+        return path_factory;
+    }
+
+    ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind) const {
+        int32_t arity = 1;
+        BinaryRow row(arity);
+        BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+        writer.WriteInt(0, 10);
+        writer.Complete();
+        return CreateManifestEntry(file_name, row, kind);
+    }
+
+    ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition,
+                                      const FileKind& kind) const {
+        return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(),
+                                   DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0);
+    }
+
+    ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition,
+                                      const FileKind& kind, const BinaryRow& min_key,
+                                      const BinaryRow& max_key, int32_t level, int32_t bucket = 0,
+                                      int32_t total_buckets = 2) const {
+        auto data_file_meta = std::make_shared(
+            file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(),
+            SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32,
+            /*schema_id=*/1, level,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/3,
+            /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt);
+        return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta);
+    }
+
+    ManifestEntry CreateManifestEntryWithFirstRowId(const std::string& file_name,
+                                                    const BinaryRow& partition,
+                                                    const FileKind& kind, int32_t bucket,
+                                                    int64_t first_row_id, int64_t row_count) const {
+        auto data_file_meta = std::make_shared(
+            file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
+            /*max_seq_no=*/32,
+            /*schema_id=*/1, /*level=*/2,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*value_stats_cols=*/std::nullopt, first_row_id,
+            /*write_cols=*/std::nullopt);
+        return ManifestEntry(kind, partition, bucket, /*total_buckets=*/2, data_file_meta);
+    }
+
+    IndexManifestEntry CreateGlobalIndexEntry(const std::string& file_name,
+                                              const BinaryRow& partition, int32_t bucket,
+                                              int64_t row_range_start,
+                                              int64_t row_range_end) const {
+        return CreateGlobalIndexEntry(file_name, partition, bucket, FileKind::Add(),
+                                      row_range_start, row_range_end);
+    }
+
+    IndexManifestEntry CreateGlobalIndexEntry(const std::string& file_name,
+                                              const BinaryRow& partition, int32_t bucket,
+                                              const FileKind& kind, int64_t row_range_start,
+                                              int64_t row_range_end) const {
+        GlobalIndexMeta global_index_meta(row_range_start, row_range_end, /*index_field_id=*/1,
+                                          /*extra_field_ids=*/std::nullopt,
+                                          std::make_shared("meta", GetDefaultPool().get()));
+        auto index_file_meta = std::make_shared(
+            "HASH", file_name, /*file_size=*/100, /*row_count=*/5,
+            /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index_meta);
+        return IndexManifestEntry(kind, partition, bucket, index_file_meta);
+    }
+
+    IndexManifestEntry CreateDvIndexEntry(const std::string& file_name, const BinaryRow& partition,
+                                          int32_t bucket) const {
+        auto index_file_meta = std::make_shared(
+            DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, file_name, /*file_size=*/100,
+            /*row_count=*/1, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt);
+        return IndexManifestEntry(FileKind::Add(), partition, bucket, index_file_meta);
+    }
+
+    BinaryRow CreateIntRow(int32_t value) const {
+        BinaryRow row(1);
+        BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+        writer.WriteInt(0, value);
+        writer.Complete();
+        return row;
+    }
+
+    arrow::FieldVector fields_;
+};
+
+TEST_F(ConflictDetectionTest, TestFileDeletionConflicts) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("f1", FileKind::Add()));
+
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("f1", FileKind::Delete()));
+
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("f2", FileKind::Add()));
+
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("f1", FileKind::Delete()));
+        changes.push_back(CreateManifestEntry("f3", FileKind::Add()));
+
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+            "Trying to delete file f1");
+    }
+    {
+        std::vector base_entries;
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("f1", FileKind::Delete()));
+
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+            "Trying to delete file f1");
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceConflicts) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    const BinaryRow partition = CreateIntRow(10);
+    std::vector base_entries;
+    base_entries.push_back(CreateManifestEntryWithFirstRowId("base-1", partition, FileKind::Add(),
+                                                             /*bucket=*/0, /*first_row_id=*/0,
+                                                             /*row_count=*/10));
+    base_entries.push_back(CreateManifestEntryWithFirstRowId("base-2", partition, FileKind::Add(),
+                                                             /*bucket=*/0, /*first_row_id=*/10,
+                                                             /*row_count=*/10));
+    std::vector changes;
+
+    ASSERT_OK(
+        CheckConflicts(detection, base_entries, changes,
+                       {CreateGlobalIndexEntry("global-index-covered", partition, /*bucket=*/0,
+                                               /*row_range_start=*/0, /*row_range_end=*/19)},
+                       Snapshot::CommitKind::Append()));
+
+    ASSERT_NOK_WITH_MSG(
+        CheckConflicts(detection, base_entries, changes,
+                       {CreateGlobalIndexEntry("global-index-missing", partition, /*bucket=*/0,
+                                               /*row_range_start=*/0, /*row_range_end=*/20)},
+                       Snapshot::CommitKind::Append()),
+        "Global index row ID existence conflict");
+}
+
+TEST_F(ConflictDetectionTest, TestDedicatedStorageRowIdRangeConflicts) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    const BinaryRow partition = CreateIntRow(10);
+    std::vector base_entries;
+    base_entries.push_back(CreateManifestEntryWithFirstRowId("data-0.orc", partition,
+                                                             FileKind::Add(), /*bucket=*/0,
+                                                             /*first_row_id=*/0,
+                                                             /*row_count=*/10));
+
+    std::vector out_of_range_dedicated_entries;
+    out_of_range_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "blob-0.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/5,
+        /*row_count=*/10));
+    ASSERT_NOK_WITH_MSG(CheckConflicts(detection, base_entries, out_of_range_dedicated_entries,
+                                       Snapshot::CommitKind::Compact()),
+                        "is not covered by one data file range");
+
+    std::vector contained_dedicated_entries;
+    contained_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "blob-1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/5,
+        /*row_count=*/4));
+    ASSERT_OK(CheckConflicts(detection, base_entries, contained_dedicated_entries,
+                             Snapshot::CommitKind::Compact()));
+
+    std::vector disjoint_data_entries;
+    disjoint_data_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "data-a.orc", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+        /*row_count=*/5));
+    disjoint_data_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "data-b.orc", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/10,
+        /*row_count=*/5));
+    std::vector spanning_dedicated_entries;
+    spanning_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "vector-0.vector.data", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/3,
+        /*row_count=*/10));
+    ASSERT_NOK_WITH_MSG(CheckConflicts(detection, disjoint_data_entries, spanning_dedicated_entries,
+                                       Snapshot::CommitKind::Compact()),
+                        "spans multiple data file ranges");
+
+    std::vector no_data_entries;
+    std::vector dedicated_only_entries;
+    dedicated_only_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "blob-only.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+        /*row_count=*/1));
+    ASSERT_NOK_WITH_MSG(CheckConflicts(detection, no_data_entries, dedicated_only_entries,
+                                       Snapshot::CommitKind::Compact()),
+                        "is not covered by one data file range");
+}
+
+TEST_F(ConflictDetectionTest, TestBucketKeepSame) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+
+    const BinaryRow partition = CreateIntRow(10);
+    {
+        ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                    /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                    /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                    /*path_factory=*/nullptr);
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   DataFileMeta::EmptyMinKey(),
+                                                   DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                                   /*bucket=*/0, /*total_buckets=*/4));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(),
+                                              DataFileMeta::EmptyMinKey(),
+                                              DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                              /*bucket=*/0, /*total_buckets=*/4));
+
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                    /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                    /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                    /*path_factory=*/nullptr);
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   DataFileMeta::EmptyMinKey(),
+                                                   DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                                   /*bucket=*/0, /*total_buckets=*/2));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(),
+                                              DataFileMeta::EmptyMinKey(),
+                                              DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                              /*bucket=*/0, /*total_buckets=*/4));
+
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+            "Total buckets of partition");
+    }
+    {
+        ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                    /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                    /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                    /*path_factory=*/nullptr);
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   DataFileMeta::EmptyMinKey(),
+                                                   DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                                   /*bucket=*/0, /*total_buckets=*/2));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(),
+                                              DataFileMeta::EmptyMinKey(),
+                                              DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                              /*bucket=*/0, /*total_buckets=*/4));
+
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                    /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                    /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                    /*path_factory=*/nullptr);
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   DataFileMeta::EmptyMinKey(),
+                                                   DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                                   /*bucket=*/0, /*total_buckets=*/2));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(),
+                                              DataFileMeta::EmptyMinKey(),
+                                              DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                              /*bucket=*/0, /*total_buckets=*/4));
+
+        ASSERT_OK(
+            CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Overwrite()));
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestBucketKeepSameHelpers) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    const BinaryRow partition = CreateIntRow(10);
+    std::vector changes;
+    changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(),
+                                          DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+                                          /*level=*/1,
+                                          /*bucket=*/0, /*total_buckets=*/4));
+    changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(),
+                                          DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+                                          /*level=*/1,
+                                          /*bucket=*/1, /*total_buckets=*/4));
+
+    std::unordered_map expected_total_buckets;
+    ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets));
+    ASSERT_EQ(1U, expected_total_buckets.size());
+    ASSERT_EQ(4, expected_total_buckets.at(partition));
+
+    std::unordered_map previous_total_buckets;
+    previous_total_buckets.emplace(partition, 4);
+    ASSERT_OK(
+        detection.CheckSameBucketByTotalBuckets(expected_total_buckets, previous_total_buckets));
+
+    std::unordered_map cached_total_buckets;
+    ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &cached_total_buckets));
+    ASSERT_TRUE(cached_total_buckets.empty());
+
+    ConflictDetection mismatch_detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                         /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                         /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                         CreatePathFactory({"f1"}));
+    ASSERT_NOK_WITH_MSG(
+        mismatch_detection.CheckSameBucketByTotalBuckets(expected_total_buckets, {{partition, 2}}),
+        "new bucket num");
+}
+
+TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                CreatePathFactory({"f1"}));
+
+    const BinaryRow partition = CreateIntRow(10);
+    std::vector changes;
+    changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(),
+                                          DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+                                          /*level=*/1,
+                                          /*bucket=*/0, /*total_buckets=*/2));
+    changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(),
+                                          DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+                                          /*level=*/1,
+                                          /*bucket=*/1, /*total_buckets=*/4));
+
+    std::unordered_map total_buckets;
+    // Verify the partition value is rendered via FileStorePathFactory::GetPartitionString
+    // (i.e. "partition {f1=10...}"), not dropped. Before the fmt escaping fix this printed a
+    // literal "partition {}" with the partition string silently discarded.
+    ASSERT_NOK_WITH_MSG(detection.CollectUncheckedBucketPartitions(changes, &total_buckets),
+                        "partition {f1=10");
+}
+
+TEST_F(ConflictDetectionTest, TestBucketKeepSameCacheEviction) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    const int32_t total_buckets = 4;
+    for (int32_t value = 0; value <= 1000; ++value) {
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", CreateIntRow(value), FileKind::Add(),
+                                              DataFileMeta::EmptyMinKey(),
+                                              DataFileMeta::EmptyMaxKey(), /*level=*/1,
+                                              /*bucket=*/0, total_buckets));
+
+        std::unordered_map expected_total_buckets;
+        ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets));
+        ASSERT_EQ(1U, expected_total_buckets.size());
+        ASSERT_OK(detection.CheckSameBucketByTotalBuckets(expected_total_buckets,
+                                                          expected_total_buckets));
+    }
+
+    std::vector evicted_partition_changes;
+    evicted_partition_changes.push_back(
+        CreateManifestEntry("delta", CreateIntRow(0), FileKind::Add(), DataFileMeta::EmptyMinKey(),
+                            DataFileMeta::EmptyMaxKey(), /*level=*/1, /*bucket=*/0, total_buckets));
+    std::unordered_map evicted_partition_buckets;
+    ASSERT_OK(detection.CollectUncheckedBucketPartitions(evicted_partition_changes,
+                                                         &evicted_partition_buckets));
+    ASSERT_EQ(1U, evicted_partition_buckets.size());
+}
+
+TEST_F(ConflictDetectionTest, TestDeletionVectorsNotSupportedWithBucketUnawareMode) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::BUCKET, "0"},
+                                               {Options::DELETION_VECTORS_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{},
+                                       Snapshot::CommitKind::Append()),
+                        "not yet support dv with BUCKET_UNAWARE mode");
+}
+
+TEST_F(ConflictDetectionTest,
+       TestDeletionVectorsNotSupportedWithResolvedBucketUnawareModeFromMinusOne) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::BUCKET, "-1"},
+                                               {Options::DELETION_VECTORS_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{},
+                                       Snapshot::CommitKind::Append()),
+                        "not yet support dv with BUCKET_UNAWARE mode");
+}
+
+TEST_F(ConflictDetectionTest, TestDeletionVectorsAllowedWithResolvedDynamicBucketMode) {
+    auto fields = {arrow::field("f0", arrow::int32(), /*nullable=*/false),
+                   arrow::field("f1", arrow::int32(), /*nullable=*/false),
+                   arrow::field("f2", arrow::int32())};
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "-1"}}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::BUCKET, "-1"},
+                                               {Options::DELETION_VECTORS_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    ASSERT_OK(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{},
+                             Snapshot::CommitKind::Append()));
+}
+
+TEST_F(ConflictDetectionTest, TestCheckLsmKeyRangeConflict) {
+    auto fields = {arrow::field("f0", arrow::int32(), /*nullable=*/false),
+                   arrow::field("f1", arrow::int32(), /*nullable=*/false),
+                   arrow::field("f2", arrow::int32())};
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "4"}}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "4"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                CreatePathFactory({"f1"}));
+
+    const BinaryRow partition = CreateIntRow(10);
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   CreateIntRow(1), CreateIntRow(3),
+                                                   /*level=*/1));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(3),
+                                              CreateIntRow(5), /*level=*/1));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+            "LSM conflicts detected");
+    }
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   CreateIntRow(1), CreateIntRow(3),
+                                                   /*level=*/1));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(4),
+                                              CreateIntRow(5), /*level=*/1));
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   CreateIntRow(1), CreateIntRow(3),
+                                                   /*level=*/0));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2),
+                                              CreateIntRow(5), /*level=*/0));
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   CreateIntRow(1), CreateIntRow(3),
+                                                   /*level=*/1, /*bucket=*/0));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2),
+                                              CreateIntRow(5), /*level=*/1,
+                                              /*bucket=*/1));
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(),
+                                                   CreateIntRow(1), CreateIntRow(3),
+                                                   /*level=*/1));
+        std::vector changes;
+        changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(),
+                                              CreateIntRow(2), CreateIntRow(5), /*level=*/1));
+        ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()));
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestShouldBeOverwriteCommit) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    std::vector add_only_entries;
+    add_only_entries.push_back(CreateManifestEntry("f1", FileKind::Add()));
+    add_only_entries.push_back(CreateManifestEntry("f2", FileKind::Add()));
+    ASSERT_FALSE(detection.ShouldBeOverwriteCommit(add_only_entries, /*append_index_files=*/{}));
+
+    ASSERT_FALSE(detection.ShouldBeOverwriteCommit(/*append_table_files=*/{},
+                                                   /*append_index_files=*/{}));
+
+    std::vector delete_entries;
+    delete_entries.push_back(CreateManifestEntry("f1", FileKind::Delete()));
+    delete_entries.push_back(CreateManifestEntry("f2", FileKind::Add()));
+    ASSERT_TRUE(detection.ShouldBeOverwriteCommit(delete_entries, /*append_index_files=*/{}));
+
+    const BinaryRow partition = CreateIntRow(10);
+    std::vector dv_index_files;
+    dv_index_files.push_back(CreateDvIndexEntry("dv1", partition, /*bucket=*/0));
+    ASSERT_TRUE(detection.ShouldBeOverwriteCommit(/*append_table_files=*/{}, dv_index_files));
+}
+
+TEST_F(ConflictDetectionTest, TestCheckRowIdExistenceNormalFiles) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+    const BinaryRow partition = CreateIntRow(10);
+
+    // No conflict: delta references the same row-id range as an existing data file.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                              /*next_row_id=*/100, Snapshot::CommitKind::Append()));
+    }
+
+    // Base data file removed: no matching range remains.
+    {
+        std::vector base_entries;
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/100, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+
+    // Base data file rewritten with a different range.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/200));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/200, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+
+    // Normal file must match exactly one data range, not span adjacent files.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2));
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/4, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+
+    // Newly appended files (firstRowId >= nextRowId) are skipped.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "new1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/100,
+            /*row_count=*/50));
+        ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                              /*next_row_id=*/100, Snapshot::CommitKind::Append()));
+    }
+
+    // Files without a pre-assigned first row id are skipped.
+    {
+        std::vector base_entries;
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntry("f1", partition, FileKind::Add()));
+        ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                              /*next_row_id=*/100, Snapshot::CommitKind::Append()));
+    }
+
+    // A null nextRowId disables row-id existence checking.
+    {
+        std::vector base_entries;
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100));
+        ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                              /*next_row_id=*/std::nullopt,
+                                              Snapshot::CommitKind::Append()));
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestCheckRowIdExistenceDedicatedFiles) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+    const BinaryRow partition = CreateIntRow(10);
+
+    // Dedicated file contained within a single data range is allowed.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+            /*row_count=*/2));
+        ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                              /*next_row_id=*/4, Snapshot::CommitKind::Append()));
+    }
+
+    // Dedicated file spanning adjacent data files is rejected.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2));
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+            /*row_count=*/4));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/4, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+
+    // Dedicated file whose range is not covered by one data file is rejected.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+            /*row_count=*/3));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/3, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+
+    // Base dedicated files are ignored when building existing data ranges.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "old.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+            /*row_count=*/2));
+        std::vector delta_entries;
+        delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+            "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0,
+            /*row_count=*/2));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflictsWithNextRowId(detection, base_entries, delta_entries,
+                                        /*next_row_id=*/2, Snapshot::CommitKind::Append()),
+            "Row ID existence conflict");
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceByPartitionAndBucket) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    const BinaryRow partition0 = CreateIntRow(0);
+    const BinaryRow partition1 = CreateIntRow(1);
+
+    // Index in partition0 but data lives in partition1 -> conflict.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId("f1", partition1, FileKind::Add(),
+                                                                 /*bucket=*/0, /*first_row_id=*/0,
+                                                                 /*row_count=*/150));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, /*delta_entries=*/{},
+                           {CreateGlobalIndexEntry("idx", partition0, /*bucket=*/0,
+                                                   /*row_range_start=*/0, /*row_range_end=*/149)},
+                           Snapshot::CommitKind::Append()),
+            "Global index row ID existence conflict");
+    }
+
+    // Index in bucket 0 but data lives in bucket 1 -> conflict.
+    {
+        std::vector base_entries;
+        base_entries.push_back(CreateManifestEntryWithFirstRowId("f1", partition0, FileKind::Add(),
+                                                                 /*bucket=*/1, /*first_row_id=*/0,
+                                                                 /*row_count=*/150));
+        ASSERT_NOK_WITH_MSG(
+            CheckConflicts(detection, base_entries, /*delta_entries=*/{},
+                           {CreateGlobalIndexEntry("idx", partition0, /*bucket=*/0,
+                                                   /*row_range_start=*/0, /*row_range_end=*/149)},
+                           Snapshot::CommitKind::Append()),
+            "Global index row ID existence conflict");
+    }
+}
+
+TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceSkipsDeleteIndexEntry) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+    const BinaryRow partition = CreateIntRow(10);
+
+    ASSERT_OK(
+        CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{},
+                       {CreateGlobalIndexEntry("idx", partition, /*bucket=*/0, FileKind::Delete(),
+                                               /*row_range_start=*/0, /*row_range_end=*/149)},
+                       Snapshot::CommitKind::Append()));
+}
+
+TEST_F(ConflictDetectionTest, TestCheckRowIdRangeConflictsAllowsAdjacentDataFiles) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector base_entries;
+    base_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2));
+    base_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2));
+    ASSERT_OK(CheckConflicts(detection, base_entries, /*delta_entries=*/{},
+                             Snapshot::CommitKind::Compact()));
+}
+
+TEST_F(ConflictDetectionTest, TestCheckRowIdRangeConflictsAllowsDedicatedFileCoveredByOneDataFile) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector base_entries;
+    base_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4));
+    std::vector delta_entries;
+    delta_entries.push_back(CreateManifestEntryWithFirstRowId(
+        "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/1, /*row_count=*/2));
+    ASSERT_OK(
+        CheckConflicts(detection, base_entries, delta_entries, Snapshot::CommitKind::Compact()));
+}
+
+TEST_F(ConflictDetectionTest, TestConflictMessageTruncatesLargeEntryList) {
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"},
+                            /*primary_keys=*/{}, /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({}));
+    ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr,
+                                /*manifest_list=*/nullptr, /*manifest_file=*/nullptr,
+                                /*commit_scanner=*/nullptr, "test_user", "test_table",
+                                /*path_factory=*/nullptr);
+
+    // kMaxEntry in conflict_detection.cpp is 50. Exceed it so the conflict message appends the
+    // "not fully displayed" truncation hint. Each lone DELETE (no matching base add) is a
+    // deletion conflict, so the delta entry list drives the message.
+    constexpr int kEntryCount = 60;
+    std::vector base_entries;
+    std::vector changes;
+    changes.reserve(kEntryCount);
+    for (int i = 0; i < kEntryCount; ++i) {
+        changes.push_back(CreateManifestEntry("delete-" + std::to_string(i), FileKind::Delete()));
+    }
+
+    ASSERT_NOK_WITH_MSG(
+        CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+        "which is not previously added");
+    ASSERT_NOK_WITH_MSG(
+        CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()),
+        "not fully displayed");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.cpp b/src/paimon/core/operation/commit/manifest_entry_changes.cpp
new file mode 100644
index 00000000..1ca54a4b
--- /dev/null
+++ b/src/paimon/core/operation/commit/manifest_entry_changes.cpp
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/manifest_entry_changes.h"
+
+#include 
+
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/io/compact_increment.h"
+#include "paimon/core/io/data_increment.h"
+
+namespace paimon {
+
+ManifestEntryChanges::ManifestEntryChanges(int32_t default_num_bucket)
+    : default_num_bucket_(default_num_bucket) {}
+
+Status ManifestEntryChanges::Collect(const std::shared_ptr& message) {
+    auto commit_message = std::dynamic_pointer_cast(message);
+    if (!commit_message) {
+        return Status::Invalid("fail to cast commit message to commit message impl");
+    }
+
+    DataIncrement new_files_increment = commit_message->GetNewFilesIncrement();
+    for (const std::shared_ptr& file : new_files_increment.NewFiles()) {
+        append_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : new_files_increment.DeletedFiles()) {
+        append_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : new_files_increment.ChangelogFiles()) {
+        append_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : new_files_increment.DeletedIndexFiles()) {
+        append_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(),
+                                        commit_message->Bucket(), file);
+    }
+    for (const std::shared_ptr& file : new_files_increment.NewIndexFiles()) {
+        append_index_files.emplace_back(FileKind::Add(), commit_message->Partition(),
+                                        commit_message->Bucket(), file);
+    }
+
+    CompactIncrement compact_increment = commit_message->GetCompactIncrement();
+    for (const std::shared_ptr& file : compact_increment.CompactBefore()) {
+        compact_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : compact_increment.CompactAfter()) {
+        compact_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : compact_increment.ChangelogFiles()) {
+        compact_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file));
+    }
+    for (const std::shared_ptr& file : compact_increment.DeletedIndexFiles()) {
+        compact_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(),
+                                         commit_message->Bucket(), file);
+    }
+    for (const std::shared_ptr& file : compact_increment.NewIndexFiles()) {
+        compact_index_files.emplace_back(FileKind::Add(), commit_message->Partition(),
+                                         commit_message->Bucket(), file);
+    }
+
+    return Status::OK();
+}
+
+bool ManifestEntryChanges::HasAppendChanges() const {
+    return !append_table_files.empty() || !append_changelog.empty() || !append_index_files.empty();
+}
+
+bool ManifestEntryChanges::HasGlobalIndexFileAdditions() const {
+    for (const IndexManifestEntry& index_entry : append_index_files) {
+        if (index_entry.kind == FileKind::Add() && index_entry.index_file->GetGlobalIndexMeta()) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool ManifestEntryChanges::HasCompactChanges() const {
+    return !compact_table_files.empty() || !compact_changelog.empty() ||
+           !compact_index_files.empty();
+}
+
+std::string ManifestEntryChanges::ToString() const {
+    std::vector parts;
+    if (!append_table_files.empty()) {
+        parts.push_back(fmt::format("{} append table files", append_table_files.size()));
+    }
+    if (!append_changelog.empty()) {
+        parts.push_back(fmt::format("{} append Changelogs", append_changelog.size()));
+    }
+    if (!append_index_files.empty()) {
+        parts.push_back(fmt::format("{} append index files", append_index_files.size()));
+    }
+    if (!compact_table_files.empty()) {
+        parts.push_back(fmt::format("{} compact table files", compact_table_files.size()));
+    }
+    if (!compact_changelog.empty()) {
+        parts.push_back(fmt::format("{} compact Changelogs", compact_changelog.size()));
+    }
+    if (!compact_index_files.empty()) {
+        parts.push_back(fmt::format("{} compact index files", compact_index_files.size()));
+    }
+    return fmt::format("{}", fmt::join(parts, ", "));
+}
+
+std::vector ManifestEntryChanges::ChangedPartitions(
+    const std::vector& data_file_changes,
+    const std::vector& index_file_changes) {
+    std::unordered_set changed_partitions;
+    for (const ManifestEntry& file : data_file_changes) {
+        changed_partitions.insert(file.Partition());
+    }
+    for (const IndexManifestEntry& file : index_file_changes) {
+        if (file.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX ||
+            file.index_file->GetGlobalIndexMeta()) {
+            changed_partitions.insert(file.partition);
+        }
+    }
+    return std::vector(changed_partitions.begin(), changed_partitions.end());
+}
+
+ManifestEntry ManifestEntryChanges::MakeEntry(
+    const FileKind& kind, const std::shared_ptr& commit_message,
+    const std::shared_ptr& file) const {
+    int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt
+                                ? default_num_bucket_
+                                : commit_message->TotalBuckets().value();
+    return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets,
+                         file);
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.h b/src/paimon/core/operation/commit/manifest_entry_changes.h
new file mode 100644
index 00000000..bafb8bf0
--- /dev/null
+++ b/src/paimon/core/operation/commit/manifest_entry_changes.h
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/commit_message.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+/// Detailed changes from `CommitMessage`s.
+class ManifestEntryChanges {
+ public:
+    explicit ManifestEntryChanges(int32_t default_num_bucket);
+
+    Status Collect(const std::shared_ptr& message);
+
+    bool HasAppendChanges() const;
+
+    bool HasGlobalIndexFileAdditions() const;
+
+    bool HasCompactChanges() const;
+
+    std::string ToString() const;
+
+    static std::vector ChangedPartitions(
+        const std::vector& data_file_changes,
+        const std::vector& index_file_changes);
+
+ public:
+    std::vector append_table_files;
+    std::vector append_changelog;
+    std::vector append_index_files;
+    std::vector compact_table_files;
+    std::vector compact_changelog;
+    std::vector compact_index_files;
+
+ private:
+    ManifestEntry MakeEntry(const FileKind& kind,
+                            const std::shared_ptr& commit_message,
+                            const std::shared_ptr& file) const;
+
+ private:
+    int32_t default_num_bucket_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp
new file mode 100644
index 00000000..434f7b4c
--- /dev/null
+++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/manifest_entry_changes.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/core/index/global_index_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/compact_increment.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_increment.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/defs.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class ManifestEntryChangesTest : public testing::Test {
+ protected:
+    BinaryRow CreateIntRow(int32_t value) const {
+        BinaryRow row(1);
+        BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+        writer.WriteInt(0, value);
+        writer.Complete();
+        return row;
+    }
+
+    std::shared_ptr CreateDataFileMeta(const std::string& file_name) const {
+        return std::make_shared(
+            file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
+            /*max_seq_no=*/32,
+            /*schema_id=*/1, /*level=*/2,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt);
+    }
+
+    std::shared_ptr CreateIndexFileMeta(
+        const std::string& file_name, const std::string& index_type = "bitmap") const {
+        return std::make_shared(
+            index_type, file_name, /*file_size=*/100, /*row_count=*/5,
+            /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt,
+            /*global_index_meta=*/std::nullopt);
+    }
+
+    std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name) const {
+        GlobalIndexMeta global_index(/*row_range_start=*/0, /*row_range_end=*/3,
+                                     /*index_field_id=*/1, /*extra_field_ids=*/std::nullopt,
+                                     std::make_shared("meta", GetDefaultPool().get()));
+        return std::make_shared(
+            "HASH", file_name, /*file_size=*/100, /*row_count=*/5,
+            /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index);
+    }
+};
+
+TEST_F(ManifestEntryChangesTest, TestCollectAndSummary) {
+    const BinaryRow partition = CreateIntRow(10);
+
+    DataIncrement data_increment(
+        {CreateDataFileMeta("append-add")}, {CreateDataFileMeta("append-del")},
+        {CreateDataFileMeta("append-changelog")}, {CreateIndexFileMeta("append-index-add")},
+        {CreateIndexFileMeta("append-index-del")});
+    CompactIncrement compact_increment(
+        {CreateDataFileMeta("compact-before")}, {CreateDataFileMeta("compact-after")},
+        {CreateDataFileMeta("compact-changelog")}, {CreateIndexFileMeta("compact-index-add")},
+        {CreateIndexFileMeta("compact-index-del")});
+
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment);
+
+    ManifestEntryChanges changes(/*default_num_bucket=*/8);
+    ASSERT_OK(changes.Collect(message));
+
+    ASSERT_EQ(2u, changes.append_table_files.size());
+    ASSERT_EQ(1u, changes.append_changelog.size());
+    ASSERT_EQ(2u, changes.append_index_files.size());
+    ASSERT_EQ(2u, changes.compact_table_files.size());
+    ASSERT_EQ(1u, changes.compact_changelog.size());
+    ASSERT_EQ(2u, changes.compact_index_files.size());
+
+    ASSERT_TRUE(changes.HasAppendChanges());
+    ASSERT_FALSE(changes.HasGlobalIndexFileAdditions());
+    ASSERT_TRUE(changes.HasCompactChanges());
+
+    ASSERT_EQ(FileKind::Add(), changes.append_table_files[0].Kind());
+    ASSERT_EQ(FileKind::Delete(), changes.append_table_files[1].Kind());
+    ASSERT_EQ(4, changes.append_table_files[0].TotalBuckets());
+
+    std::string summary = changes.ToString();
+    ASSERT_NE(std::string::npos, summary.find("2 append table files"));
+    ASSERT_NE(std::string::npos, summary.find("1 append Changelogs"));
+    ASSERT_NE(std::string::npos, summary.find("2 compact index files"));
+}
+
+TEST_F(ManifestEntryChangesTest, TestHasGlobalIndexFileAdditions) {
+    const BinaryRow partition = CreateIntRow(10);
+
+    DataIncrement data_increment(
+        /*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{},
+        /*new_index_files=*/{CreateGlobalIndexFileMeta("append-global-index")},
+        /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{},
+                                       /*changelog_files=*/{},
+                                       /*new_index_files=*/{},
+                                       /*deleted_index_files=*/{});
+
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment);
+
+    ManifestEntryChanges changes(/*default_num_bucket=*/8);
+    ASSERT_OK(changes.Collect(message));
+
+    ASSERT_TRUE(changes.HasGlobalIndexFileAdditions());
+}
+
+TEST_F(ManifestEntryChangesTest, TestCollectInvalidCommitMessageType) {
+    ManifestEntryChanges changes(/*default_num_bucket=*/8);
+    std::shared_ptr invalid_message = std::make_shared();
+    ASSERT_NOK_WITH_MSG(changes.Collect(invalid_message),
+                        "fail to cast commit message to commit message impl");
+}
+
+TEST_F(ManifestEntryChangesTest, TestChangedPartitionsIncludesDvAndGlobalIndex) {
+    const BinaryRow partition_data = CreateIntRow(10);
+    const BinaryRow partition_dv = CreateIntRow(20);
+    const BinaryRow partition_global = CreateIntRow(30);
+    const BinaryRow partition_plain_index = CreateIntRow(40);
+
+    std::vector data_changes;
+    data_changes.emplace_back(FileKind::Add(), partition_data, /*bucket=*/0, /*total_buckets=*/2,
+                              CreateDataFileMeta("data-file"));
+
+    std::vector index_changes;
+    index_changes.emplace_back(FileKind::Add(), partition_dv, /*bucket=*/0,
+                               CreateIndexFileMeta("dv-file", "DELETION_VECTORS"));
+    index_changes.emplace_back(FileKind::Add(), partition_global, /*bucket=*/0,
+                               CreateGlobalIndexFileMeta("global-index"));
+    index_changes.emplace_back(FileKind::Add(), partition_plain_index, /*bucket=*/0,
+                               CreateIndexFileMeta("plain-index", "bitmap"));
+
+    std::vector changed =
+        ManifestEntryChanges::ChangedPartitions(data_changes, index_changes);
+
+    auto contains = [&changed](const BinaryRow& target) {
+        return std::find(changed.begin(), changed.end(), target) != changed.end();
+    };
+
+    ASSERT_TRUE(contains(partition_data));
+    ASSERT_TRUE(contains(partition_dv));
+    ASSERT_TRUE(contains(partition_global));
+    ASSERT_FALSE(contains(partition_plain_index));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp
new file mode 100644
index 00000000..e8c02c9f
--- /dev/null
+++ b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/overwrite_changes_provider.h"
+
+#include 
+
+#include "paimon/core/manifest/file_kind.h"
+
+namespace paimon {
+
+OverwriteChangesProvider::OverwriteChangesProvider(std::vector changes,
+                                                   std::vector index_entries,
+                                                   ManifestScan manifest_scan, IndexScan index_scan)
+    : changes_(std::move(changes)),
+      index_entries_(std::move(index_entries)),
+      manifest_scan_(std::move(manifest_scan)),
+      index_scan_(std::move(index_scan)) {}
+
+Result> OverwriteChangesProvider::Provide(
+    const std::optional& latest_snapshot) const {
+    std::vector delta_files;
+    std::vector changelog_files;
+    std::vector index_entries = index_entries_;
+
+    if (!latest_snapshot) {
+        delta_files.insert(delta_files.end(), changes_.begin(), changes_.end());
+        return std::make_shared(std::move(delta_files), std::move(changelog_files),
+                                               std::move(index_entries));
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector entries,
+                           manifest_scan_(latest_snapshot.value()));
+    for (const auto& entry : entries) {
+        delta_files.emplace_back(FileKind::Delete(), entry.Partition(), entry.Bucket(),
+                                 entry.TotalBuckets(), entry.File());
+    }
+
+    delta_files.insert(delta_files.end(), changes_.begin(), changes_.end());
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector previous_index_entries,
+                           index_scan_(latest_snapshot.value()));
+    for (const auto& entry : previous_index_entries) {
+        index_entries.emplace_back(FileKind::Delete(), entry.partition, entry.bucket,
+                                   entry.index_file);
+    }
+
+    return std::make_shared(std::move(delta_files), std::move(changelog_files),
+                                           std::move(index_entries));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.h b/src/paimon/core/operation/commit/overwrite_changes_provider.h
new file mode 100644
index 00000000..b63170a7
--- /dev/null
+++ b/src/paimon/core/operation/commit/overwrite_changes_provider.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+
+namespace paimon {
+
+class OverwriteChangesProvider final : public CommitChangesProvider {
+ public:
+    using ManifestScan =
+        std::function>(const Snapshot& snapshot)>;
+    using IndexScan =
+        std::function>(const Snapshot& snapshot)>;
+
+    OverwriteChangesProvider(std::vector changes,
+                             std::vector index_entries,
+                             ManifestScan manifest_scan, IndexScan index_scan);
+
+    Result> Provide(
+        const std::optional& latest_snapshot) const override;
+
+ private:
+    std::vector changes_;
+    std::vector index_entries_;
+    ManifestScan manifest_scan_;
+    IndexScan index_scan_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp
new file mode 100644
index 00000000..8130fe65
--- /dev/null
+++ b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/overwrite_changes_provider.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+BinaryRow CreateIntRow(int32_t value) {
+    BinaryRow row(1);
+    BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+    writer.WriteInt(0, value);
+    writer.Complete();
+    return row;
+}
+
+ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind,
+                                  int32_t partition_value, int32_t level = 0) {
+    auto file_meta = std::make_shared(
+        file_name, /*file_size=*/1024, /*row_count=*/8, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_seq_no=*/0,
+        /*max_seq_no=*/0,
+        /*schema_id=*/0, level,
+        /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(0, 0),
+        /*delete_row_count=*/std::nullopt,
+        /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+        /*value_stats_cols=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1,
+                         file_meta);
+}
+
+IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value,
+                                    const FileKind& kind = FileKind::Add()) {
+    auto index_file = std::make_shared(
+        /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1,
+        /*dv_ranges=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*global_index_meta=*/std::nullopt);
+    return IndexManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, index_file);
+}
+
+Snapshot MakeSnapshot() {
+    return Snapshot(
+        /*id=*/1,
+        /*schema_id=*/0,
+        /*base_manifest_list=*/"base-manifest-list",
+        /*base_manifest_list_size=*/std::nullopt,
+        /*delta_manifest_list=*/"delta-manifest-list",
+        /*delta_manifest_list_size=*/std::nullopt,
+        /*changelog_manifest_list=*/std::nullopt,
+        /*changelog_manifest_list_size=*/std::nullopt,
+        /*index_manifest=*/std::nullopt,
+        /*commit_user=*/"test-user",
+        /*commit_identifier=*/1, Snapshot::CommitKind::Overwrite(),
+        /*time_millis=*/0,
+        /*total_record_count=*/0,
+        /*delta_record_count=*/0,
+        /*changelog_record_count=*/std::nullopt,
+        /*watermark=*/std::nullopt,
+        /*statistics=*/std::nullopt,
+        /*properties=*/std::nullopt,
+        /*next_row_id=*/std::nullopt);
+}
+
+}  // namespace
+
+TEST(OverwriteChangesProviderTest, TestProvideWithoutLatestSnapshotUsesChangesOnly) {
+    std::vector changes = {
+        CreateManifestEntry("new-1", FileKind::Add(), /*partition_value=*/1),
+        CreateManifestEntry("new-2", FileKind::Delete(), /*partition_value=*/1)};
+    std::vector index_entries = {
+        CreateIndexEntry("index-new", /*partition_value=*/1)};
+
+    int manifest_scan_calls = 0;
+    int index_scan_calls = 0;
+    OverwriteChangesProvider provider(
+        changes, index_entries,
+        [&manifest_scan_calls](const Snapshot&) -> Result> {
+            ++manifest_scan_calls;
+            return Status::Invalid("should not call manifest_scan");
+        },
+        [&index_scan_calls](const Snapshot&) -> Result> {
+            ++index_scan_calls;
+            return Status::Invalid("should not call index_scan");
+        });
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider.Provide(std::nullopt));
+
+    const std::vector& delta_files = provided->delta_files;
+    const std::vector& changelog_files = provided->changelog_files;
+    const std::vector& provided_index_entries = provided->index_entries;
+
+    ASSERT_EQ(0, manifest_scan_calls);
+    ASSERT_EQ(0, index_scan_calls);
+    ASSERT_TRUE(changelog_files.empty());
+    ASSERT_EQ(changes.size(), delta_files.size());
+    ASSERT_EQ(index_entries.size(), provided_index_entries.size());
+    EXPECT_EQ("new-1", delta_files[0].FileName());
+    EXPECT_EQ("new-2", delta_files[1].FileName());
+    EXPECT_EQ("index-new", provided_index_entries[0].index_file->FileName());
+}
+
+TEST(OverwriteChangesProviderTest, TestProvideWithLatestSnapshotAddsDeletesAndAppendsAllChanges) {
+    std::vector changes = {
+        CreateManifestEntry("existing-a", FileKind::Add(), /*partition_value=*/1),
+        CreateManifestEntry("new-b", FileKind::Add(), /*partition_value=*/1),
+        CreateManifestEntry("force-delete-c", FileKind::Delete(), /*partition_value=*/1)};
+    std::vector index_entries = {
+        CreateIndexEntry("index-new", /*partition_value=*/1)};
+
+    OverwriteChangesProvider provider(
+        changes, index_entries,
+        [](const Snapshot&) -> Result> {
+            return std::vector{
+                CreateManifestEntry("existing-a", FileKind::Add(), /*partition_value=*/1),
+                CreateManifestEntry("existing-x", FileKind::Add(), /*partition_value=*/1)};
+        },
+        [](const Snapshot&) -> Result> {
+            return std::vector{
+                CreateIndexEntry("index-old", /*partition_value=*/1)};
+        });
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr provided,
+                         provider.Provide(std::optional(MakeSnapshot())));
+
+    const std::vector& delta_files = provided->delta_files;
+    const std::vector& changelog_files = provided->changelog_files;
+    const std::vector& provided_index_entries = provided->index_entries;
+
+    ASSERT_TRUE(changelog_files.empty());
+
+    // delta = delete(existing-a), delete(existing-x),
+    //         add(existing-a), add(new-b), delete(force-delete-c)
+    ASSERT_EQ(5u, delta_files.size());
+    EXPECT_TRUE(delta_files[0].Kind() == FileKind::Delete());
+    EXPECT_EQ("existing-a", delta_files[0].FileName());
+    EXPECT_TRUE(delta_files[1].Kind() == FileKind::Delete());
+    EXPECT_EQ("existing-x", delta_files[1].FileName());
+    EXPECT_TRUE(delta_files[2].Kind() == FileKind::Add());
+    EXPECT_EQ("existing-a", delta_files[2].FileName());
+    EXPECT_TRUE(delta_files[3].Kind() == FileKind::Add());
+    EXPECT_EQ("new-b", delta_files[3].FileName());
+    EXPECT_TRUE(delta_files[4].Kind() == FileKind::Delete());
+    EXPECT_EQ("force-delete-c", delta_files[4].FileName());
+
+    // index = provided new + delete old
+    ASSERT_EQ(2u, provided_index_entries.size());
+    EXPECT_TRUE(provided_index_entries[0].kind == FileKind::Add());
+    EXPECT_EQ("index-new", provided_index_entries[0].index_file->FileName());
+    EXPECT_TRUE(provided_index_entries[1].kind == FileKind::Delete());
+    EXPECT_EQ("index-old", provided_index_entries[1].index_file->FileName());
+}
+
+TEST(OverwriteChangesProviderTest, TestProvidePropagatesScanErrors) {
+    std::vector changes = {
+        CreateManifestEntry("new-1", FileKind::Add(), /*partition_value=*/1)};
+    std::vector index_entries;
+
+    OverwriteChangesProvider provider_manifest_fail(
+        changes, index_entries,
+        [](const Snapshot&) -> Result> {
+            return Status::Invalid("manifest scan failed");
+        },
+        [](const Snapshot&) -> Result> {
+            return std::vector{};
+        });
+
+    ASSERT_NOK_WITH_MSG(provider_manifest_fail.Provide(std::optional(MakeSnapshot())),
+                        "manifest scan failed");
+
+    OverwriteChangesProvider provider_index_fail(
+        changes, index_entries,
+        [](const Snapshot&) -> Result> {
+            return std::vector{};
+        },
+        [](const Snapshot&) -> Result> {
+            return Status::Invalid("index scan failed");
+        });
+
+    ASSERT_NOK_WITH_MSG(provider_index_fail.Provide(std::optional(MakeSnapshot())),
+                        "index scan failed");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/retry_waiter.cpp b/src/paimon/core/operation/commit/retry_waiter.cpp
new file mode 100644
index 00000000..94422225
--- /dev/null
+++ b/src/paimon/core/operation/commit/retry_waiter.cpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/retry_waiter.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace paimon {
+
+RetryWaiter::RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms)
+    : min_retry_wait_ms_(std::max(0, min_retry_wait_ms)),
+      max_retry_wait_ms_(std::max(0, max_retry_wait_ms)) {}
+
+void RetryWaiter::RetryWait(int32_t retry_count) const {
+    int32_t non_negative_retry_count = std::max(0, retry_count);
+    double exponential = std::pow(2.0, static_cast(non_negative_retry_count));
+    int64_t retry_wait = 0;
+    if (min_retry_wait_ms_ > 0 && max_retry_wait_ms_ > 0) {
+        double max_safe_exponential =
+            static_cast(max_retry_wait_ms_) / static_cast(min_retry_wait_ms_);
+        if (!std::isfinite(exponential) || exponential >= max_safe_exponential) {
+            retry_wait = max_retry_wait_ms_;
+        } else {
+            retry_wait = static_cast(min_retry_wait_ms_ * exponential);
+        }
+    }
+
+    int64_t jitter_upper = std::max(1, static_cast(retry_wait * 0.2));
+    std::mt19937 rng(std::random_device{}());  // NOLINT(whitespace/braces)
+    std::uniform_int_distribution dist(0, jitter_upper - 1);
+    retry_wait += dist(rng);
+
+    if (retry_wait <= 0) {
+        return;
+    }
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(retry_wait));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/retry_waiter.h b/src/paimon/core/operation/commit/retry_waiter.h
new file mode 100644
index 00000000..c1e08305
--- /dev/null
+++ b/src/paimon/core/operation/commit/retry_waiter.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+
+namespace paimon {
+
+class RetryWaiter {
+ public:
+    RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms);
+
+    void RetryWait(int32_t retry_count) const;
+
+ private:
+    int64_t min_retry_wait_ms_;
+    int64_t max_retry_wait_ms_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/retry_waiter_test.cpp b/src/paimon/core/operation/commit/retry_waiter_test.cpp
new file mode 100644
index 00000000..2d8ea105
--- /dev/null
+++ b/src/paimon/core/operation/commit/retry_waiter_test.cpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/retry_waiter.h"
+
+#include 
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+
+TEST(RetryWaiterTest, TestRetryWaitWithZeroBoundsReturnsQuickly) {
+    RetryWaiter waiter(/*min_retry_wait_ms=*/0, /*max_retry_wait_ms=*/0);
+
+    auto begin = std::chrono::steady_clock::now();
+    waiter.RetryWait(/*retry_count=*/3);
+    auto elapsed = std::chrono::duration_cast(
+        std::chrono::steady_clock::now() - begin);
+
+    ASSERT_LT(elapsed.count(), 20);
+}
+
+TEST(RetryWaiterTest, TestRetryWaitWithLargeRetryCountIsClamped) {
+    RetryWaiter waiter(/*min_retry_wait_ms=*/10, /*max_retry_wait_ms=*/10);
+
+    auto begin = std::chrono::steady_clock::now();
+    waiter.RetryWait(/*retry_count=*/1024);
+    auto elapsed = std::chrono::duration_cast(
+        std::chrono::steady_clock::now() - begin);
+
+    ASSERT_GE(elapsed.count(), 8);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp
new file mode 100644
index 00000000..d805233f
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/row_id_column_conflict_checker.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/range_helper.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+Result> RowIdColumnConflictChecker::FromDataFiles(
+    const std::shared_ptr& schema_manager,
+    const std::vector>& delta_files) {
+    auto checker =
+        std::shared_ptr(new RowIdColumnConflictChecker(schema_manager));
+
+    PAIMON_RETURN_NOT_OK(checker->BuildWriteRanges(delta_files));
+    return checker;
+}
+
+Status RowIdColumnConflictChecker::BuildWriteRanges(
+    const std::vector>& delta_files) {
+    std::vector> row_id_files;
+    row_id_files.reserve(delta_files.size());
+    for (const auto& file : delta_files) {
+        if (file && file->first_row_id.has_value()) {
+            row_id_files.push_back(file);
+        }
+    }
+
+    if (row_id_files.empty()) {
+        write_ranges_.clear();
+        return Status::OK();
+    }
+
+    // 1. merge overlapping ranges and calculate [Range, unordered_set] struct.
+    RangeHelper> range_helper(
+        [](const std::shared_ptr& file) -> Result {
+            return file->first_row_id.value();
+        },
+        [](const std::shared_ptr& file) -> Result {
+            return file->first_row_id.value() + file->row_count - 1;
+        });
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector>> merged_range_groups,
+        range_helper.MergeOverlappingRanges(std::move(row_id_files)));
+
+    write_ranges_.clear();
+
+    for (const auto& group : merged_range_groups) {
+        Range merged_range = MergeRange(group);
+        std::unordered_set field_ids;
+
+        for (const auto& file : group) {
+            PAIMON_RETURN_NOT_OK(AddWriteFieldIds(file, &field_ids));
+        }
+
+        write_ranges_.push_back(WriteRange{merged_range, std::move(field_ids)});
+    }
+
+    // 2. sort by range for binary search
+    std::sort(write_ranges_.begin(), write_ranges_.end(),
+              [](const WriteRange& a, const WriteRange& b) {
+                  if (a.range.from != b.range.from) {
+                      return a.range.from < b.range.from;
+                  }
+                  return a.range.to < b.range.to;
+              });
+
+    return Status::OK();
+}
+
+Range RowIdColumnConflictChecker::MergeRange(
+    const std::vector>& files) const {
+    int64_t from = std::numeric_limits::max();
+    int64_t to = std::numeric_limits::min();
+    for (const auto& file : files) {
+        const int64_t file_from = file->first_row_id.value();
+        const int64_t file_to = file_from + file->row_count - 1;
+        from = std::min(from, file_from);
+        to = std::max(to, file_to);
+    }
+    return Range(from, to);
+}
+
+Status RowIdColumnConflictChecker::AddWriteFieldIds(const std::shared_ptr& file,
+                                                    std::unordered_set* field_ids) {
+    if (!file->write_cols.has_value()) {
+        std::map field_id_by_name;
+        PAIMON_ASSIGN_OR_RAISE(field_id_by_name, FieldIdByName(file->schema_id));
+        for (const auto& entry : field_id_by_name) {
+            field_ids->insert(entry.second);
+        }
+        return Status::OK();
+    }
+
+    for (const auto& write_col : file->write_cols.value()) {
+        PAIMON_ASSIGN_OR_RAISE(std::optional field_id, FieldId(file, write_col));
+        if (field_id.has_value()) {
+            field_ids->insert(field_id.value());
+        }
+    }
+
+    return Status::OK();
+}
+
+Result RowIdColumnConflictChecker::ConflictsWith(
+    const std::shared_ptr& file) const {
+    if (!file->first_row_id.has_value()) {
+        return false;
+    }
+
+    Range range(file->first_row_id.value(), file->first_row_id.value() + file->row_count - 1);
+    int32_t index = FirstPossibleRange(range);
+    while (index < static_cast(write_ranges_.size())) {
+        const auto& write_range = write_ranges_[index];
+        if (write_range.range.from > range.to) {
+            return false;
+        }
+        // overlapping row range and overlapping write fields
+        if (Range::HasIntersection(write_range.range, range)) {
+            PAIMON_ASSIGN_OR_RAISE(bool has_common_write_field,
+                                   ContainsAnyWriteField(write_range.field_ids, file));
+            if (has_common_write_field) {
+                return true;
+            }
+        }
+        ++index;
+    }
+
+    return false;
+}
+
+int32_t RowIdColumnConflictChecker::FirstPossibleRange(const Range& range) const {
+    int32_t low = 0;
+    auto high = static_cast(write_ranges_.size());
+    while (low < high) {
+        const int32_t mid = low + (high - low) / 2;
+        if (write_ranges_[mid].range.to < range.from) {
+            low = mid + 1;
+        } else {
+            high = mid;
+        }
+    }
+    return low;
+}
+
+Result RowIdColumnConflictChecker::ContainsAnyWriteField(
+    const std::unordered_set& field_ids, const std::shared_ptr& file) const {
+    // If write cols == null, it's a full-schema write
+    if (!file->write_cols.has_value()) {
+        return true;
+    }
+
+    for (const auto& write_col : file->write_cols.value()) {
+        PAIMON_ASSIGN_OR_RAISE(std::optional field_id, FieldId(file, write_col));
+        if (field_id.has_value() && field_ids.count(field_id.value()) > 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+Result> RowIdColumnConflictChecker::FieldId(
+    const std::shared_ptr& file, const std::string& write_col) const {
+    std::map field_id_by_name;
+    PAIMON_ASSIGN_OR_RAISE(field_id_by_name, FieldIdByName(file->schema_id));
+    auto it = field_id_by_name.find(write_col);
+    if (it != field_id_by_name.end()) {
+        return std::optional(it->second);
+    }
+
+    if (SpecialFields::IsSystemField(write_col)) {
+        return std::optional();
+    }
+
+    return Status::Invalid("Cannot find write column '" + write_col + "' in schema " +
+                           std::to_string(file->schema_id) + ".");
+}
+
+Result> RowIdColumnConflictChecker::FieldIdByName(
+    int64_t schema_id) const {
+    auto cache_it = field_id_by_name_cache_.find(schema_id);
+    if (cache_it != field_id_by_name_cache_.end()) {
+        return cache_it->second;
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema,
+                           schema_manager_->ReadSchema(schema_id));
+    std::map mapping;
+    for (const auto& field : schema->Fields()) {
+        mapping[field.Name()] = field.Id();
+    }
+
+    auto [it, inserted] = field_id_by_name_cache_.emplace(schema_id, std::move(mapping));
+    (void)inserted;
+    return it->second;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.h b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h
new file mode 100644
index 00000000..0576ed74
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/utils/range.h"
+
+namespace paimon {
+
+/// Detects row-id range conflicts only when written field ids overlap. The detection process is as
+/// below:
+///
+///   Merge delta files by row range and calculate updated columns.
+///   Sort those items by range.
+///   For each checking files, do binary search to find overlapping ranges. If their updated
+///   columns also overlap, return conflicting result.
+///
+class RowIdColumnConflictChecker {
+ public:
+    static Result> FromDataFiles(
+        const std::shared_ptr& schema_manager,
+        const std::vector>& delta_files);
+
+    bool IsEmpty() const {
+        return write_ranges_.empty();
+    }
+
+    /// Check whether a committed incremental file entry conflicts with current committing delta
+    /// files. If an existing file has both overlapping row range and overlapping write fields, then
+    /// it conflicts.
+    ///
+    /// @param file committed incremental data file
+    /// @return true if conflict
+    Result ConflictsWith(const std::shared_ptr& file) const;
+
+ private:
+    /// Range and field id Set.
+    struct WriteRange {
+        Range range;
+        std::unordered_set field_ids;
+    };
+
+    explicit RowIdColumnConflictChecker(const std::shared_ptr& schema_manager)
+        : schema_manager_(schema_manager) {}
+
+    Status BuildWriteRanges(const std::vector>& delta_files);
+    Status AddWriteFieldIds(const std::shared_ptr& file,
+                            std::unordered_set* field_ids);
+    Range MergeRange(const std::vector>& files) const;
+    int32_t FirstPossibleRange(const Range& range) const;
+    Result ContainsAnyWriteField(const std::unordered_set& field_ids,
+                                       const std::shared_ptr& file) const;
+    Result> FieldId(const std::shared_ptr& file,
+                                           const std::string& write_col) const;
+    Result> FieldIdByName(int64_t schema_id) const;
+
+ private:
+    std::shared_ptr schema_manager_;
+    std::vector write_ranges_;
+    mutable std::map> field_id_by_name_cache_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp
new file mode 100644
index 00000000..e9091b0f
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/row_id_column_conflict_checker.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class RowIdColumnConflictCheckerTest : public testing::Test {
+ protected:
+    void SetUp() override {
+        auto fs = std::make_shared();
+        const std::string table_root =
+            GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/";
+        schema_manager_ = std::make_shared(fs, table_root);
+    }
+
+    std::shared_ptr CreateFile(
+        const std::string& file_name, std::optional first_row_id, int64_t row_count,
+        int64_t schema_id, std::optional> write_cols) const {
+        return std::make_shared(
+            file_name, /*file_size=*/0, row_count, DataFileMeta::EmptyMinKey(),
+            DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+            /*min_seq_no=*/0,
+            /*max_seq_no=*/0, schema_id, /*level=*/0,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*first_row_id=*/first_row_id, write_cols);
+    }
+
+    Result> CreateChecker(
+        const std::vector>& files) const {
+        return RowIdColumnConflictChecker::FromDataFiles(schema_manager_, files);
+    }
+
+ private:
+    std::shared_ptr schema_manager_;
+};
+
+TEST_F(RowIdColumnConflictCheckerTest, TestAllowsDisjointWriteColumns) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10,
+                                                /*schema_id=*/0, std::vector{"b"})}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"c"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical));
+    ASSERT_FALSE(conflicts);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestDetectsSameWriteColumns) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10,
+                                                /*schema_id=*/0, std::vector{"b"})}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"b"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical));
+    ASSERT_TRUE(conflicts);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestUsesFieldIdAcrossRename) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10,
+                                                /*schema_id=*/1, std::vector{"c"})}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"b"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical));
+    ASSERT_TRUE(conflicts);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestTreatsNullWriteColumnsAsFullSchemaWrite) {
+    ASSERT_OK_AND_ASSIGN(auto checker,
+                         CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10,
+                                                   /*schema_id=*/0, /*write_cols=*/std::nullopt)}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"b"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical));
+    ASSERT_TRUE(conflicts);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestMergesOverlappedDeltaRangesAndWriteColumns) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current-b", /*first_row_id=*/0, /*row_count=*/11,
+                                                /*schema_id=*/0, std::vector{"b"}),
+                                     CreateFile("current-c", /*first_row_id=*/5, /*row_count=*/11,
+                                                /*schema_id=*/0, std::vector{"c"})}));
+
+    auto historical_b = CreateFile("historical-b", /*first_row_id=*/12, /*row_count=*/1,
+                                   /*schema_id=*/0, std::vector{"b"});
+    auto historical_c = CreateFile("historical-c", /*first_row_id=*/12, /*row_count=*/1,
+                                   /*schema_id=*/0, std::vector{"c"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts_b, checker->ConflictsWith(historical_b));
+    ASSERT_OK_AND_ASSIGN(bool conflicts_c, checker->ConflictsWith(historical_c));
+    ASSERT_TRUE(conflicts_b);
+    ASSERT_TRUE(conflicts_c);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestScansAllOverlappedRangesAfterBinarySearch) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current-b", /*first_row_id=*/0, /*row_count=*/5,
+                                                /*schema_id=*/0, std::vector{"b"}),
+                                     CreateFile("current-c", /*first_row_id=*/10, /*row_count=*/5,
+                                                /*schema_id=*/0, std::vector{"c"})}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/3, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"c"});
+    ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical));
+    ASSERT_TRUE(conflicts);
+}
+
+TEST_F(RowIdColumnConflictCheckerTest, TestIgnoreUnknownNonSystemWriteColumn) {
+    ASSERT_OK_AND_ASSIGN(
+        auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10,
+                                                /*schema_id=*/0, std::vector{"b"})}));
+
+    auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10,
+                                 /*schema_id=*/0, std::vector{"missing"});
+    auto conflicts = checker->ConflictsWith(historical);
+    ASSERT_FALSE(conflicts.ok());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp
new file mode 100644
index 00000000..1a337705
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/row_tracking_commit_utils.h"
+
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/vector_store_utils.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+namespace {
+
+ManifestEntry CloneEntryWithClonedFileMeta(const ManifestEntry& entry) {
+    auto cloned_file = std::make_shared(*entry.File());
+    return ManifestEntry(entry.Kind(), entry.Partition(), entry.Bucket(), entry.TotalBuckets(),
+                         cloned_file);
+}
+
+}  // namespace
+
+Result RowTrackingCommitUtils::AssignRowTracking(
+    int64_t new_snapshot_id, int64_t first_row_id_start,
+    const std::vector& delta_files) {
+    // assigned snapshot id to delta files
+    std::vector snapshot_assigned;
+    AssignSnapshotId(new_snapshot_id, delta_files, &snapshot_assigned);
+    // assign row id for new files
+    std::vector row_id_assigned;
+    PAIMON_ASSIGN_OR_RAISE(
+        int64_t next_row_id_start,
+        AssignRowTrackingMeta(first_row_id_start, snapshot_assigned, &row_id_assigned));
+    return RowTrackingAssigned{next_row_id_start, std::move(row_id_assigned)};
+}
+
+void RowTrackingCommitUtils::AssignSnapshotId(int64_t snapshot_id,
+                                              const std::vector& delta_files,
+                                              std::vector* snapshot_assigned) {
+    for (const auto& entry : delta_files) {
+        ManifestEntry assigned_entry = CloneEntryWithClonedFileMeta(entry);
+        int64_t min_seq_number = assigned_entry.File()->min_sequence_number;
+        int64_t max_seq_number = assigned_entry.File()->max_sequence_number;
+        if (min_seq_number == 0L) {
+            // Case 1: New file (e.g., from INSERT)
+            // All records in this file get the current snapshot ID as sequence number
+            assigned_entry.AssignSequenceNumber(snapshot_id, snapshot_id);
+        } else if (max_seq_number == 0L) {
+            // Case 2: File with some modified records
+            // - min: preserve original sequence number (from unmodified records)
+            // - max: assign current snapshot ID
+            assigned_entry.AssignSequenceNumber(min_seq_number, snapshot_id);
+        } else {
+            // Case 3: Pure compact file (no modified records)
+            // Preserve original min/max sequence numbers from source files.
+        }
+        snapshot_assigned->emplace_back(std::move(assigned_entry));
+    }
+}
+
+Result RowTrackingCommitUtils::AssignRowTrackingMeta(
+    int64_t first_row_id_start, const std::vector& delta_files,
+    std::vector* row_id_assigned) {
+    if (delta_files.empty()) {
+        return first_row_id_start;
+    }
+    // assign row id for new files
+    int64_t start = first_row_id_start;
+    int64_t blob_start_default = first_row_id_start;
+    std::map blob_starts;
+    int64_t vector_store_start = first_row_id_start;
+
+    for (const auto& entry : delta_files) {
+        ManifestEntry assigned_entry = CloneEntryWithClonedFileMeta(entry);
+        if (!entry.File()->file_source) {
+            return Status::Invalid(
+                "This is a bug, file source field for row-tracking table must present.");
+        }
+
+        bool contains_row_id =
+            entry.File()->write_cols.has_value() &&
+            std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(),
+                      SpecialFields::RowId().Name()) != entry.File()->write_cols->end();
+
+        if (entry.File()->file_source.value() == FileSource::Append() &&
+            entry.File()->first_row_id == std::nullopt && !contains_row_id) {
+            int64_t row_count = entry.File()->row_count;
+            if (BlobUtils::IsBlobFile(entry.File()->file_name)) {
+                if (!entry.File()->write_cols || entry.File()->write_cols->empty()) {
+                    return Status::Invalid(fmt::format(
+                        "invalid blob file {}: does not have write_cols", entry.File()->file_name));
+                }
+                std::string blob_field_name = entry.File()->write_cols->at(0);
+                int64_t blob_start = blob_starts.count(blob_field_name)
+                                         ? blob_starts[blob_field_name]
+                                         : blob_start_default;
+                if (blob_start >= start) {
+                    return Status::Invalid(
+                        fmt::format("This is a bug, blobStart {} should be less than start {} when "
+                                    "assigning a blob entry file.",
+                                    blob_start, start));
+                }
+                assigned_entry.AssignFirstRowId(blob_start);
+                blob_starts[blob_field_name] = blob_start + row_count;
+            } else if (VectorStoreUtils::IsVectorStoreFile(entry.File()->file_name)) {
+                if (vector_store_start >= start) {
+                    return Status::Invalid(fmt::format(
+                        "This is a bug, vectorStoreStart {} should be less than start {} "
+                        "when assigning a vector-store entry file.",
+                        vector_store_start, start));
+                }
+                assigned_entry.AssignFirstRowId(vector_store_start);
+                vector_store_start += row_count;
+            } else {
+                assigned_entry.AssignFirstRowId(start);
+                blob_start_default = start;
+                blob_starts.clear();
+                start += row_count;
+            }
+        } else {
+            // for compact file, do not assign first row id.
+        }
+        row_id_assigned->emplace_back(std::move(assigned_entry));
+    }
+    return start;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.h b/src/paimon/core/operation/commit/row_tracking_commit_utils.h
new file mode 100644
index 00000000..766d36a7
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+/// Utils for row tracking commit.
+class RowTrackingCommitUtils {
+ public:
+    struct RowTrackingAssigned {
+        int64_t next_row_id_start;
+        std::vector assigned_entries;
+    };
+
+    // Assign sequence numbers and row ids for row-tracking commit.
+    static Result AssignRowTracking(
+        int64_t new_snapshot_id, int64_t first_row_id_start,
+        const std::vector& delta_files);
+
+ private:
+    static void AssignSnapshotId(int64_t snapshot_id, const std::vector& delta_files,
+                                 std::vector* snapshot_assigned);
+
+    static Result AssignRowTrackingMeta(int64_t first_row_id_start,
+                                                 const std::vector& delta_files,
+                                                 std::vector* row_id_assigned);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp
new file mode 100644
index 00000000..02e79fc5
--- /dev/null
+++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/row_tracking_commit_utils.h"
+
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class RowTrackingCommitUtilsTest : public testing::Test {
+ protected:
+    BinaryRow CreateIntRow(int32_t value) const {
+        BinaryRow row(1);
+        BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+        writer.WriteInt(0, value);
+        writer.Complete();
+        return row;
+    }
+
+    ManifestEntry CreateEntry(const std::string& file_name, int64_t row_count,
+                              int64_t min_seq_number, int64_t max_seq_number,
+                              const std::optional& file_source,
+                              const std::optional>& write_cols) const {
+        auto file_meta = std::make_shared(
+            file_name, /*file_size=*/row_count, row_count, DataFileMeta::EmptyMinKey(),
+            DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+            min_seq_number, max_seq_number,
+            /*schema_id=*/1, /*level=*/0,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, file_source,
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*first_row_id=*/std::nullopt, write_cols);
+        return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1,
+                             file_meta);
+    }
+
+    ManifestEntry CreateEntryWithFirstRowId(
+        const std::string& file_name, int64_t row_count, int64_t min_seq_number,
+        int64_t max_seq_number, const std::optional& file_source,
+        const std::optional>& write_cols,
+        const std::optional& first_row_id) const {
+        auto file_meta = std::make_shared(
+            file_name, /*file_size=*/row_count, row_count, DataFileMeta::EmptyMinKey(),
+            DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+            min_seq_number, max_seq_number,
+            /*schema_id=*/1, /*level=*/0,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, file_source,
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, first_row_id, write_cols);
+        return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1,
+                             file_meta);
+    }
+};
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequence) {
+    std::vector input;
+    input.push_back(CreateEntry("new-file", /*row_count=*/10, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+    input.push_back(CreateEntry("partial-modified", /*row_count=*/8, /*min_seq_number=*/7,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+    input.push_back(CreateEntry("compact-file", /*row_count=*/6, /*min_seq_number=*/3,
+                                /*max_seq_number=*/5, FileSource::Compact(),
+                                std::vector{"f0"}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input));
+
+    ASSERT_EQ(3u, assigned.assigned_entries.size());
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number);
+    EXPECT_EQ(7, assigned.assigned_entries[1].File()->min_sequence_number);
+    EXPECT_EQ(100, assigned.assigned_entries[1].File()->max_sequence_number);
+    EXPECT_EQ(3, assigned.assigned_entries[2].File()->min_sequence_number);
+    EXPECT_EQ(5, assigned.assigned_entries[2].File()->max_sequence_number);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequenceRangeStartingAtZero) {
+    std::vector input;
+    input.push_back(CreateEntry("range-starts-at-zero", /*row_count=*/10,
+                                /*min_seq_number=*/0, /*max_seq_number=*/9, FileSource::Append(),
+                                std::vector{"f0"}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input));
+
+    ASSERT_EQ(1u, assigned.assigned_entries.size());
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTracking) {
+    std::vector input;
+    input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+    input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"blob_a"}));
+    input.push_back(CreateEntry("blob-a-2.blob", /*row_count=*/2, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"blob_a"}));
+    input.push_back(CreateEntry("vector-1.vector.data", /*row_count=*/4,
+                                /*min_seq_number=*/0, /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"vec"}));
+    input.push_back(CreateEntry("normal-file-2", /*row_count=*/5, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/200, /*first_row_id_start=*/0, input));
+
+    ASSERT_EQ(5u, assigned.assigned_entries.size());
+    EXPECT_EQ(0, assigned.assigned_entries[0].File()->first_row_id.value());
+    EXPECT_EQ(0, assigned.assigned_entries[1].File()->first_row_id.value());
+    EXPECT_EQ(3, assigned.assigned_entries[2].File()->first_row_id.value());
+    EXPECT_EQ(0, assigned.assigned_entries[3].File()->first_row_id.value());
+    EXPECT_EQ(10, assigned.assigned_entries[4].File()->first_row_id.value());
+    EXPECT_EQ(15, assigned.next_row_id_start);
+
+    for (const auto& entry : assigned.assigned_entries) {
+        EXPECT_EQ(200, entry.File()->min_sequence_number);
+        EXPECT_EQ(200, entry.File()->max_sequence_number);
+    }
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingWithoutFileSource) {
+    std::vector input;
+    input.push_back(CreateEntry("invalid-no-source", /*row_count=*/1, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, std::nullopt,
+                                std::vector{"f0"}));
+
+    ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking(
+                            /*new_snapshot_id=*/1, /*first_row_id_start=*/0, input),
+                        "file source field for row-tracking table must present");
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingDoesNotMutateInputEntries) {
+    std::vector input;
+    input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+    input.push_back(CreateEntry("normal-file-2", /*row_count=*/5, /*min_seq_number=*/7,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+
+    std::shared_ptr input_file_0 = input[0].File();
+    std::shared_ptr input_file_1 = input[1].File();
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/200, /*first_row_id_start=*/1000, input));
+
+    ASSERT_EQ(2u, assigned.assigned_entries.size());
+
+    EXPECT_EQ(0, input[0].File()->min_sequence_number);
+    EXPECT_EQ(0, input[0].File()->max_sequence_number);
+    EXPECT_EQ(std::nullopt, input[0].File()->first_row_id);
+
+    EXPECT_EQ(7, input[1].File()->min_sequence_number);
+    EXPECT_EQ(0, input[1].File()->max_sequence_number);
+    EXPECT_EQ(std::nullopt, input[1].File()->first_row_id);
+
+    EXPECT_NE(input_file_0.get(), assigned.assigned_entries[0].File().get());
+    EXPECT_NE(input_file_1.get(), assigned.assigned_entries[1].File().get());
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingReassignsOnRetryWithAdvancedRowId) {
+    std::vector input;
+    input.push_back(CreateEntry("retry-file", /*row_count=*/10, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned first_attempt,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/1000, input));
+    ASSERT_EQ(1u, first_attempt.assigned_entries.size());
+    EXPECT_EQ(100, first_attempt.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(100, first_attempt.assigned_entries[0].File()->max_sequence_number);
+    ASSERT_TRUE(first_attempt.assigned_entries[0].File()->first_row_id.has_value());
+    EXPECT_EQ(1000, first_attempt.assigned_entries[0].File()->first_row_id.value());
+    EXPECT_EQ(1010, first_attempt.next_row_id_start);
+
+    // Simulate CAS retry with a newer latest snapshot and advanced next_row_id.
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned second_attempt,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/101, /*first_row_id_start=*/2000, input));
+    ASSERT_EQ(1u, second_attempt.assigned_entries.size());
+    EXPECT_EQ(101, second_attempt.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(101, second_attempt.assigned_entries[0].File()->max_sequence_number);
+    ASSERT_TRUE(second_attempt.assigned_entries[0].File()->first_row_id.has_value());
+    EXPECT_EQ(2000, second_attempt.assigned_entries[0].File()->first_row_id.value());
+    EXPECT_EQ(2010, second_attempt.next_row_id_start);
+
+    // Input remains immutable across attempts; retry assignment always starts from fresh metadata.
+    EXPECT_EQ(0, input[0].File()->min_sequence_number);
+    EXPECT_EQ(0, input[0].File()->max_sequence_number);
+    EXPECT_EQ(std::nullopt, input[0].File()->first_row_id);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingEmptyInput) {
+    std::vector input;
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/42, input));
+    EXPECT_TRUE(assigned.assigned_entries.empty());
+    EXPECT_EQ(42, assigned.next_row_id_start);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingSkipsFilesWithRowIdColumn) {
+    std::vector input;
+    input.push_back(
+        CreateEntry("has-row-id-column", /*row_count=*/10, /*min_seq_number=*/0,
+                    /*max_seq_number=*/0, FileSource::Append(),
+                    std::vector{std::string(SpecialFields::RowId().Name())}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input));
+
+    ASSERT_EQ(1u, assigned.assigned_entries.size());
+    // Sequence numbers are still stamped for a new file.
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number);
+    // But a file already carrying the row-id column must not be assigned a first row id.
+    EXPECT_EQ(std::nullopt, assigned.assigned_entries[0].File()->first_row_id);
+    EXPECT_EQ(0, assigned.next_row_id_start);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingKeepsExistingFirstRowId) {
+    std::vector input;
+    input.push_back(CreateEntryWithFirstRowId("already-assigned", /*row_count=*/10,
+                                              /*min_seq_number=*/0, /*max_seq_number=*/0,
+                                              FileSource::Append(), std::vector{"f0"},
+                                              /*first_row_id=*/500));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input));
+
+    ASSERT_EQ(1u, assigned.assigned_entries.size());
+    // The existing first row id is preserved and start is not advanced.
+    ASSERT_TRUE(assigned.assigned_entries[0].File()->first_row_id.has_value());
+    EXPECT_EQ(500, assigned.assigned_entries[0].File()->first_row_id.value());
+    EXPECT_EQ(0, assigned.next_row_id_start);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingCompactFileKeepsNoFirstRowId) {
+    std::vector input;
+    input.push_back(CreateEntry("compact-file", /*row_count=*/6, /*min_seq_number=*/3,
+                                /*max_seq_number=*/5, FileSource::Compact(),
+                                std::vector{"f0"}));
+
+    ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                         RowTrackingCommitUtils::AssignRowTracking(
+                             /*new_snapshot_id=*/100, /*first_row_id_start=*/7, input));
+
+    ASSERT_EQ(1u, assigned.assigned_entries.size());
+    // Pure compact file keeps its original sequence numbers and gets no first row id.
+    EXPECT_EQ(3, assigned.assigned_entries[0].File()->min_sequence_number);
+    EXPECT_EQ(5, assigned.assigned_entries[0].File()->max_sequence_number);
+    EXPECT_EQ(std::nullopt, assigned.assigned_entries[0].File()->first_row_id);
+    EXPECT_EQ(7, assigned.next_row_id_start);
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingBlobBeforeNormalFileFails) {
+    std::vector input;
+    input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"blob_a"}));
+
+    ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking(
+                            /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input),
+                        "blobStart");
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingVectorStoreBeforeNormalFileFails) {
+    std::vector input;
+    input.push_back(CreateEntry("vector-1.vector.data", /*row_count=*/4, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"vec"}));
+
+    ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking(
+                            /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input),
+                        "vectorStoreStart");
+}
+
+TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingBlobWithoutWriteColsFails) {
+    std::vector input;
+    input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{"f0"}));
+    input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0,
+                                /*max_seq_number=*/0, FileSource::Append(),
+                                std::vector{}));
+
+    ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking(
+                            /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input),
+                        "does not have write_cols");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp
new file mode 100644
index 00000000..51729b99
--- /dev/null
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/sequence_snapshot_properties.h"
+
+#include 
+
+#include 
+#include 
+
+#include "paimon/core/manifest/file_kind.h"
+
+namespace paimon {
+
+Result> SequenceSnapshotProperties::MaxSequenceNumber(
+    const std::optional& snapshot) {
+    if (!snapshot || !snapshot.value().Properties()) {
+        return std::optional();
+    }
+
+    const auto& properties = snapshot.value().Properties().value();
+    auto iter = properties.find(kMaxSequenceNumberKey);
+    if (iter == properties.end()) {
+        return std::optional();
+    }
+
+    try {
+        size_t parsed = 0;
+        int64_t value = std::stoll(iter->second, &parsed);
+        if (parsed != iter->second.size()) {
+            return Status::Invalid(
+                fmt::format("Invalid {} value '{}': trailing characters are not allowed",
+                            kMaxSequenceNumberKey, iter->second));
+        }
+        return std::optional(value);
+    } catch (const std::exception& e) {
+        return Status::Invalid(fmt::format("Invalid {} value '{}': {}", kMaxSequenceNumberKey,
+                                           iter->second, e.what()));
+    }
+}
+
+std::optional SequenceSnapshotProperties::MaxSequenceNumberFromFiles(
+    const std::vector& files) {
+    int64_t max_sequence_number = std::numeric_limits::min();
+    bool found = false;
+    for (const auto& file : files) {
+        if (!(file.Kind() == FileKind::Add())) {
+            continue;
+        }
+        max_sequence_number = std::max(max_sequence_number, file.File()->max_sequence_number);
+        found = true;
+    }
+
+    if (!found) {
+        return std::nullopt;
+    }
+    return max_sequence_number;
+}
+
+std::map SequenceSnapshotProperties::MergeMaxSequenceNumber(
+    const std::map& properties,
+    const std::optional& latest_max_sequence_number,
+    const std::vector& delta_files) {
+    std::map snapshot_properties = properties;
+
+    std::optional delta_max_sequence_number = MaxSequenceNumberFromFiles(delta_files);
+    if (delta_max_sequence_number || latest_max_sequence_number) {
+        int64_t merged_max_sequence_number = latest_max_sequence_number
+                                                 ? latest_max_sequence_number.value()
+                                                 : delta_max_sequence_number.value();
+        if (delta_max_sequence_number) {
+            merged_max_sequence_number =
+                std::max(merged_max_sequence_number, delta_max_sequence_number.value());
+        }
+        snapshot_properties[kMaxSequenceNumberKey] = std::to_string(merged_max_sequence_number);
+    }
+
+    return snapshot_properties;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.h b/src/paimon/core/operation/commit/sequence_snapshot_properties.h
new file mode 100644
index 00000000..06beac30
--- /dev/null
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+class ManifestFile;
+class ManifestFileMeta;
+
+class SequenceSnapshotProperties {
+ public:
+    SequenceSnapshotProperties() = delete;
+
+    static constexpr const char* kMaxSequenceNumberKey = "sequence.generation.max-sequence-number";
+
+    static Result> MaxSequenceNumber(
+        const std::optional& snapshot);
+
+    static std::optional MaxSequenceNumberFromFiles(
+        const std::vector& files);
+
+    static std::map MergeMaxSequenceNumber(
+        const std::map& properties,
+        const std::optional& latest_max_sequence_number,
+        const std::vector& delta_files);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
new file mode 100644
index 00000000..af572b72
--- /dev/null
+++ b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/commit/sequence_snapshot_properties.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class SequenceSnapshotPropertiesTest : public testing::Test {
+ protected:
+    Snapshot MakeSnapshot(
+        const std::optional>& properties) const {
+        return Snapshot(
+            /*id=*/1,
+            /*schema_id=*/1,
+            /*base_manifest_list=*/"base-manifest-list",
+            /*base_manifest_list_size=*/std::nullopt,
+            /*delta_manifest_list=*/"delta-manifest-list",
+            /*delta_manifest_list_size=*/std::nullopt,
+            /*changelog_manifest_list=*/std::nullopt,
+            /*changelog_manifest_list_size=*/std::nullopt,
+            /*index_manifest=*/std::nullopt,
+            /*commit_user=*/"test-user",
+            /*commit_identifier=*/1, Snapshot::CommitKind::Append(),
+            /*time_millis=*/0,
+            /*total_record_count=*/0,
+            /*delta_record_count=*/0,
+            /*changelog_record_count=*/std::nullopt,
+            /*watermark=*/std::nullopt,
+            /*statistics=*/std::nullopt, properties,
+            /*next_row_id=*/std::nullopt);
+    }
+
+    std::shared_ptr CreateDataFileMeta(int64_t max_sequence_number) const {
+        return std::make_shared(
+            "data-file", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/0,
+            /*max_seq_no=*/max_sequence_number,
+            /*schema_id=*/1, /*level=*/0,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, /*file_source=*/std::nullopt,
+            /*external_path=*/std::nullopt,
+            /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt);
+    }
+
+    ManifestEntry CreateEntry(const FileKind& kind, int64_t max_sequence_number) const {
+        return ManifestEntry(kind, BinaryRow(0), /*bucket=*/0, /*total_buckets=*/1,
+                             CreateDataFileMeta(max_sequence_number));
+    }
+};
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberEmptySnapshot) {
+    ASSERT_OK_AND_ASSIGN(std::optional result,
+                         SequenceSnapshotProperties::MaxSequenceNumber(std::nullopt));
+    ASSERT_FALSE(result.has_value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberSnapshotWithoutProperties) {
+    ASSERT_OK_AND_ASSIGN(std::optional result,
+                         SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(std::nullopt)));
+    ASSERT_FALSE(result.has_value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberKeyMissing) {
+    std::map properties{{"other-key", "42"}};
+    ASSERT_OK_AND_ASSIGN(std::optional result,
+                         SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)));
+    ASSERT_FALSE(result.has_value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberValid) {
+    std::map properties{
+        {SequenceSnapshotProperties::kMaxSequenceNumberKey, "123"}};
+    ASSERT_OK_AND_ASSIGN(std::optional result,
+                         SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)));
+    ASSERT_TRUE(result.has_value());
+    ASSERT_EQ(123, result.value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberTrailingCharacters) {
+    std::map properties{
+        {SequenceSnapshotProperties::kMaxSequenceNumberKey, "123abc"}};
+    ASSERT_NOK_WITH_MSG(SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)),
+                        "trailing characters are not allowed");
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberNotANumber) {
+    std::map properties{
+        {SequenceSnapshotProperties::kMaxSequenceNumberKey, "not-a-number"}};
+    ASSERT_NOK_WITH_MSG(SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)),
+                        "Invalid");
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesEmpty) {
+    ASSERT_FALSE(SequenceSnapshotProperties::MaxSequenceNumberFromFiles({}).has_value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesOnlyDelete) {
+    std::vector files{CreateEntry(FileKind::Delete(), 100)};
+    ASSERT_FALSE(SequenceSnapshotProperties::MaxSequenceNumberFromFiles(files).has_value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesSkipsDelete) {
+    std::vector files{CreateEntry(FileKind::Add(), 10),
+                                     CreateEntry(FileKind::Delete(), 999),
+                                     CreateEntry(FileKind::Add(), 42)};
+    std::optional result = SequenceSnapshotProperties::MaxSequenceNumberFromFiles(files);
+    ASSERT_TRUE(result.has_value());
+    ASSERT_EQ(42, result.value());
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberNoInput) {
+    std::map properties{{"existing", "value"}};
+    std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+        properties, /*latest_max_sequence_number=*/std::nullopt, /*delta_files=*/{});
+    ASSERT_EQ(properties, merged);
+    ASSERT_EQ(0u, merged.count(SequenceSnapshotProperties::kMaxSequenceNumberKey));
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberLatestOnly) {
+    std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+        /*properties=*/{}, /*latest_max_sequence_number=*/50, /*delta_files=*/{});
+    ASSERT_EQ("50", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey));
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberDeltaOnly) {
+    std::vector delta_files{CreateEntry(FileKind::Add(), 77)};
+    std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+        /*properties=*/{}, /*latest_max_sequence_number=*/std::nullopt, delta_files);
+    ASSERT_EQ("77", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey));
+}
+
+TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberTakesMaximum) {
+    std::vector delta_files{CreateEntry(FileKind::Add(), 30)};
+    std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+        /*properties=*/{}, /*latest_max_sequence_number=*/90, delta_files);
+    ASSERT_EQ("90", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey));
+
+    std::vector larger_delta{CreateEntry(FileKind::Add(), 150)};
+    std::map merged2 = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+        /*properties=*/{}, /*latest_max_sequence_number=*/90, larger_delta);
+    ASSERT_EQ("150", merged2.at(SequenceSnapshotProperties::kMaxSequenceNumberKey));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp
index 490a0756..fe98095d 100644
--- a/src/paimon/core/operation/commit_context.cpp
+++ b/src/paimon/core/operation/commit_context.cpp
@@ -31,6 +31,7 @@ namespace paimon {
 
 CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user,
                              bool ignore_empty_commit, bool use_rest_catalog_commit,
+                             bool append_commit_check_conflict,
                              const std::shared_ptr& memory_pool,
                              const std::shared_ptr& executor,
                              const std::shared_ptr& specific_file_system,
@@ -39,6 +40,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co
       commit_user_(commit_user),
       ignore_empty_commit_(ignore_empty_commit),
       use_rest_catalog_commit_(use_rest_catalog_commit),
+      append_commit_check_conflict_(append_commit_check_conflict),
       memory_pool_(memory_pool),
       executor_(executor),
       specific_file_system_(specific_file_system),
@@ -53,6 +55,7 @@ class CommitContextBuilder::Impl {
     void Reset() {
         ignore_empty_commit_ = true;
         use_rest_catalog_commit_ = false;
+        append_commit_check_conflict_ = false;
         memory_pool_ = GetDefaultPool();
         executor_ = CreateDefaultExecutor();
         specific_file_system_.reset();
@@ -64,6 +67,7 @@ class CommitContextBuilder::Impl {
     std::string commit_user_;
     bool ignore_empty_commit_ = true;
     bool use_rest_catalog_commit_ = false;
+    bool append_commit_check_conflict_ = false;
     std::shared_ptr memory_pool_ = GetDefaultPool();
     std::shared_ptr executor_ = CreateDefaultExecutor();
     std::shared_ptr specific_file_system_;
@@ -101,6 +105,12 @@ CommitContextBuilder& CommitContextBuilder::UseRESTCatalogCommit(bool use_rest_c
     return *this;
 }
 
+CommitContextBuilder& CommitContextBuilder::AppendCommitCheckConflict(
+    bool append_commit_check_conflict) {
+    impl_->append_commit_check_conflict_ = append_commit_check_conflict;
+    return *this;
+}
+
 CommitContextBuilder& CommitContextBuilder::WithMemoryPool(
     const std::shared_ptr& memory_pool) {
     impl_->memory_pool_ = memory_pool;
@@ -126,8 +136,8 @@ Result> CommitContextBuilder::Finish() {
     }
     auto ctx = std::make_unique(
         impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_,
-        impl_->use_rest_catalog_commit_, impl_->memory_pool_, impl_->executor_,
-        impl_->specific_file_system_, impl_->options_);
+        impl_->use_rest_catalog_commit_, impl_->append_commit_check_conflict_, impl_->memory_pool_,
+        impl_->executor_, impl_->specific_file_system_, impl_->options_);
     impl_->Reset();
     return ctx;
 }
diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp
new file mode 100644
index 00000000..e790958f
--- /dev/null
+++ b/src/paimon/core/operation/commit_context_test.cpp
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/commit_context.h"
+
+#include "gtest/gtest.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/executor.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_system.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(CommitContextTest, TestDefaultValue) {
+    CommitContextBuilder builder("table_root_path", "commit_user_1");
+
+    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path"));
+
+    ASSERT_EQ(ctx->GetRootPath(), expected_root_path);
+    ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1");
+    ASSERT_TRUE(ctx->IgnoreEmptyCommit());
+    ASSERT_FALSE(ctx->UseRESTCatalogCommit());
+    ASSERT_FALSE(ctx->AppendCommitCheckConflict());
+    ASSERT_TRUE(ctx->GetMemoryPool());
+    ASSERT_TRUE(ctx->GetExecutor());
+    ASSERT_FALSE(ctx->GetSpecificFileSystem());
+    ASSERT_TRUE(ctx->GetOptions().empty());
+}
+
+TEST(CommitContextTest, TestSetContent) {
+    CommitContextBuilder builder("table_root_path", "commit_user_1");
+
+    auto memory_pool = GetDefaultPool();
+    std::shared_ptr executor = CreateDefaultExecutor();
+    auto fs = std::make_shared();
+
+    ASSERT_OK_AND_ASSIGN(auto ctx, builder.IgnoreEmptyCommit(false)
+                                       .UseRESTCatalogCommit(true)
+                                       .AppendCommitCheckConflict(true)
+                                       .WithMemoryPool(memory_pool)
+                                       .WithExecutor(executor)
+                                       .WithFileSystem(fs)
+                                       .AddOption("key", "value")
+                                       .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path"));
+    ASSERT_EQ(ctx->GetRootPath(), expected_root_path);
+    ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1");
+    ASSERT_FALSE(ctx->IgnoreEmptyCommit());
+    ASSERT_TRUE(ctx->UseRESTCatalogCommit());
+    ASSERT_TRUE(ctx->AppendCommitCheckConflict());
+    ASSERT_EQ(ctx->GetMemoryPool(), memory_pool);
+    ASSERT_EQ(ctx->GetExecutor(), executor);
+    ASSERT_EQ(ctx->GetSpecificFileSystem(), fs);
+
+    std::map expected_options = {{"key", "value"}};
+    ASSERT_EQ(ctx->GetOptions(), expected_options);
+}
+
+TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) {
+    CommitContextBuilder builder("table_root_path", "commit_user_1");
+    builder.AddOption("old", "value");
+    builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}});
+
+    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
+
+    std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}};
+    ASSERT_EQ(ctx->GetOptions(), expected_options);
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/commit_metrics_test.cpp b/src/paimon/core/operation/commit_metrics_test.cpp
index c478f599..240d9134 100644
--- a/src/paimon/core/operation/commit_metrics_test.cpp
+++ b/src/paimon/core/operation/commit_metrics_test.cpp
@@ -18,15 +18,56 @@
 
 #include "paimon/core/operation/metrics/commit_metrics.h"
 
+#include 
 #include 
+#include 
 #include 
+#include 
 
 #include "gtest/gtest.h"
+#include "paimon/common/data/binary_row_writer.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/operation/metrics/commit_stats.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
 
+namespace {
+
+BinaryRow CreateIntRow(int32_t value) {
+    BinaryRow row(1);
+    BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+    writer.WriteInt(0, value);
+    writer.Complete();
+    return row;
+}
+
+ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucket,
+                          int64_t row_count, int64_t file_size, const std::string& file_name) {
+    BinaryRow part = CreateIntRow(partition);
+    auto file_meta = std::make_shared(
+        file_name, file_size, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+        SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/0,
+        /*schema_id=*/1, /*level=*/0,
+        /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(0, 0),
+        /*delete_row_count=*/std::nullopt,
+        /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta);
+}
+
+}  // namespace
+
 TEST(CommitMetricsTest, TestSimple) {
     auto commit_metrics = std::make_shared();
     commit_metrics->SetCounter("some_metric", 100);
@@ -48,4 +89,86 @@ TEST(CommitMetricsTest, TestSimple) {
     ASSERT_EQ(200, counter);
 }
 
+TEST(CommitMetricsTest, TestReportCommitFromStats) {
+    auto metrics = std::make_shared();
+
+    std::vector append_table_files;
+    append_table_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 201, 1001, "a1"));
+    append_table_files.push_back(CreateEntry(FileKind::Delete(), 2, 3, 302, 1002, "a2"));
+
+    std::vector append_changelog_files;
+    append_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 202, 2001, "c1"));
+    append_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 301, 2002, "c2"));
+
+    std::vector compact_table_files;
+    compact_table_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 203, 3001, "k1"));
+    compact_table_files.push_back(CreateEntry(FileKind::Delete(), 3, 5, 106, 3002, "k2"));
+
+    std::vector compact_changelog_files;
+    compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 205, 4001, "ck1"));
+    compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 307, 4002, "ck2"));
+
+    CommitStats stats(append_table_files, append_changelog_files, compact_table_files,
+                      compact_changelog_files,
+                      /*commit_duration=*/3000, /*generated_snapshots=*/2, /*attempts=*/4,
+                      /*last_committed_snapshot_id=*/10);
+    CommitMetrics::ReportCommit(metrics, stats);
+
+    ASSERT_OK_AND_ASSIGN(auto last_commit_duration,
+                         metrics->GetCounter(CommitMetrics::LAST_COMMIT_DURATION));
+    EXPECT_EQ(3000, last_commit_duration);
+    ASSERT_OK_AND_ASSIGN(auto last_commit_attempts,
+                         metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
+    EXPECT_EQ(4, last_commit_attempts);
+    ASSERT_OK_AND_ASSIGN(auto last_table_files_added,
+                         metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED));
+    EXPECT_EQ(2, last_table_files_added);
+    ASSERT_OK_AND_ASSIGN(auto last_table_files_deleted,
+                         metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED));
+    EXPECT_EQ(2, last_table_files_deleted);
+    ASSERT_OK_AND_ASSIGN(auto last_table_files_appended,
+                         metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED));
+    EXPECT_EQ(2, last_table_files_appended);
+    ASSERT_OK_AND_ASSIGN(auto last_table_files_compacted,
+                         metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED));
+    EXPECT_EQ(2, last_table_files_compacted);
+    ASSERT_OK_AND_ASSIGN(auto last_changelog_files_appended,
+                         metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED));
+    EXPECT_EQ(2, last_changelog_files_appended);
+    ASSERT_OK_AND_ASSIGN(auto last_changelog_files_compacted,
+                         metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED));
+    EXPECT_EQ(2, last_changelog_files_compacted);
+    ASSERT_OK_AND_ASSIGN(auto last_generated_snapshots,
+                         metrics->GetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS));
+    EXPECT_EQ(2, last_generated_snapshots);
+    ASSERT_OK_AND_ASSIGN(auto last_delta_records_appended,
+                         metrics->GetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED));
+    EXPECT_EQ(503, last_delta_records_appended);
+    ASSERT_OK_AND_ASSIGN(auto last_changelog_records_appended,
+                         metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED));
+    EXPECT_EQ(503, last_changelog_records_appended);
+    ASSERT_OK_AND_ASSIGN(auto last_delta_records_compacted,
+                         metrics->GetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED));
+    EXPECT_EQ(309, last_delta_records_compacted);
+    ASSERT_OK_AND_ASSIGN(
+        auto last_changelog_records_compacted,
+        metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED));
+    EXPECT_EQ(512, last_changelog_records_compacted);
+    ASSERT_OK_AND_ASSIGN(auto last_partitions_written,
+                         metrics->GetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN));
+    EXPECT_EQ(3, last_partitions_written);
+    ASSERT_OK_AND_ASSIGN(auto last_buckets_written,
+                         metrics->GetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN));
+    EXPECT_EQ(3, last_buckets_written);
+    ASSERT_OK_AND_ASSIGN(auto last_compaction_input_file_size,
+                         metrics->GetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE));
+    EXPECT_EQ(3002, last_compaction_input_file_size);
+    ASSERT_OK_AND_ASSIGN(auto last_compaction_output_file_size,
+                         metrics->GetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE));
+    EXPECT_EQ(3001, last_compaction_output_file_size);
+    ASSERT_OK_AND_ASSIGN(auto last_committed_snapshot_id,
+                         metrics->GetCounter(CommitMetrics::LAST_COMMITTED_SNAPSHOT_ID));
+    EXPECT_EQ(10, last_committed_snapshot_id);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp
index 81c19c65..033d3e07 100644
--- a/src/paimon/core/operation/data_evolution_split_read.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read.cpp
@@ -18,87 +18,94 @@
 
 #include "paimon/core/operation/data_evolution_split_read.h"
 
+#include 
+#include 
 #include 
+#include 
+#include 
+#include 
+#include 
 #include 
 
+#include "arrow/array/array_base.h"
+#include "arrow/array/array_binary.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/catalog/catalog_context.h"
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/blob_view_struct.h"
 #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h"
 #include "paimon/common/global_index/complete_index_score_batch_reader.h"
+#include "paimon/common/reader/blob_fallback_batch_reader.h"
+#include "paimon/common/reader/blob_view_resolving_batch_reader.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/range_helper.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/utils/blob_view_lookup.h"
 namespace paimon {
+int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const {
+    if (files_.empty()) {
+        return 0;
+    }
+    if (!has_row_ids_selection_) {
+        // Add enforces the union range to be contiguous
+        return union_end_row_id_ - union_first_row_id_;
+    }
+    // with a row-ids selection the scan may have pruned files, leaving holes in the union
+    int64_t row_count = 0;
+    for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) {
+        row_count += range.Count();
+    }
+    return row_count;
+}
+
 Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) {
     if (!BlobUtils::IsBlobFile(file->file_name)) {
         return Status::Invalid("Only blob file can be added to a blob bunch.");
     }
     PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
-    if (first_row_id == latest_first_row_id_) {
-        if (file->max_sequence_number >= latest_max_sequence_number_) {
-            return Status::Invalid(
-                "Blob file with same first row id should have decreasing sequence number.");
-        }
-        // for files with the same first row id, file with larger sequence_number will be chosen,
-        // other files will be skipped
-        return Status::OK();
-    }
     if (!files_.empty()) {
-        if (has_row_ids_selection_) {
-            // for the case:
-            // snapshot 1: blob0 [0, 9]
-            // snapshot 2: blob1 [0, 4] + blob2 [5, 9]
-            // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as
-            // blob1 has no intersect with {5}
-            // BlobBunch will first add blob0 [0, 9]
-            // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence
-            // number
-            if (first_row_id < expected_next_first_row_id_) {
-                if (file->max_sequence_number > latest_max_sequence_number_) {
-                    row_count_ -= files_.back()->row_count;
-                    files_.pop_back();
-                } else {
-                    return Status::OK();
-                }
-            }
-        } else {
-            if (first_row_id < expected_next_first_row_id_) {
-                if (file->max_sequence_number >= latest_max_sequence_number_) {
-                    return Status::Invalid(
-                        "Blob file with overlapping row id should have decreasing sequence "
-                        "number.");
-                }
-                // for files with overlapping, if the file with smaller sequence_number is chosen,
-                // there will not exist file with larger sequence_number
-                return Status::OK();
-            } else if (first_row_id > expected_next_first_row_id_) {
-                return Status::Invalid(
-                    fmt::format("Blob file first row id should be continuous, expect {} but got {}",
-                                expected_next_first_row_id_, first_row_id));
-            }
-        }
-        if (!files_.empty()) {
-            if (file->schema_id != files_[0]->schema_id) {
-                return Status::Invalid("All files in a blob bunch should have the same schema id.");
-            }
-            if (file->write_cols != files_[0]->write_cols) {
-                return Status::Invalid(
-                    "All files in a blob bunch should have the same write columns.");
-            }
+        // Blob files for the same field may span schema ids.
+        if (file->write_cols != files_[0]->write_cols) {
+            return Status::Invalid("All files in a blob bunch should have the same write columns.");
         }
     }
-    row_count_ += file->row_count;
-    if (row_count_ > expected_row_count_) {
+    // files sharing a max sequence number form one layer, whose row id ranges must be disjoint;
+    // overlaps across layers are the expected shape of partial updates and are kept for the
+    // row-level placeholder fallback
+    auto [layer_iter, layer_inserted] =
+        sequence_group_end_.try_emplace(file->max_sequence_number, 0);
+    if (!layer_inserted && first_row_id < layer_iter->second) {
+        return Status::Invalid(fmt::format(
+            "Blob files with the same max sequence number should not have overlapping row id "
+            "ranges: file {} (max sequence number {}) starts at row id {} before the previous "
+            "file's end {}",
+            file->file_name, file->max_sequence_number, first_row_id, layer_iter->second));
+    }
+    if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) {
+        // a hole no layer covers cannot be aligned with the data files
         return Status::Invalid(
-            fmt::format("Blob files row count exceed the expect {}", expected_row_count_));
+            fmt::format("Blob file first row id should be continuous, expect {} but got {}",
+                        union_end_row_id_, first_row_id));
     }
+    int64_t end_row_id = first_row_id + file->row_count;
+    layer_iter->second = end_row_id;
+    union_first_row_id_ = std::min(union_first_row_id_, first_row_id);
+    union_end_row_id_ = std::max(union_end_row_id_, end_row_id);
+    ranges_.emplace_back(first_row_id, end_row_id - 1);
     files_.push_back(file);
-    latest_max_sequence_number_ = file->max_sequence_number;
-    latest_first_row_id_ = first_row_id;
-    expected_next_first_row_id_ = latest_first_row_id_ + file->row_count;
+    if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) {
+        return Status::Invalid(
+            fmt::format("Blob files row count exceed the expect {}", expected_row_count_));
+    }
     return Status::OK();
 }
 DataEvolutionSplitRead::DataEvolutionSplitRead(
@@ -115,28 +122,170 @@ bool DataEvolutionSplitRead::HasIndexScoreField(const std::shared_ptrGetFieldIndex(SpecialFields::IndexScore().Name()) != -1;
 }
 
+std::vector DataEvolutionSplitRead::HasBlobViewField(
+    const CoreOptions& options, const std::shared_ptr& read_schema) {
+    std::vector read_blob_view_fields;
+    std::vector blob_view_fields = options.GetBlobViewFields();
+    for (const auto& blob : blob_view_fields) {
+        if (read_schema->GetFieldByName(blob)) {
+            read_blob_view_fields.push_back(blob);
+        }
+    }
+    return read_blob_view_fields;
+}
+
 Result> DataEvolutionSplitRead::CreateReader(
     const std::shared_ptr& split) {
     if (auto indexed_split = std::dynamic_pointer_cast(split)) {
         PAIMON_RETURN_NOT_OK(indexed_split->Validate());
-        PAIMON_ASSIGN_OR_RAISE(
-            std::unique_ptr batch_reader,
-            InnerCreateReader(indexed_split->GetDataSplit(), indexed_split->RowRanges()));
+        const auto& data_split = indexed_split->GetDataSplit();
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader,
+                               InnerCreateReader(data_split, indexed_split->RowRanges()));
         if (HasIndexScoreField(raw_read_schema_)) {
-            return std::make_unique(std::move(batch_reader),
-                                                                   indexed_split->Scores(), pool_);
+            batch_reader = std::make_unique(
+                std::move(batch_reader), indexed_split->Scores(), pool_);
         }
-        return batch_reader;
+        return WrapWithBlobViewResolverIfNeeded(data_split, std::move(batch_reader));
     } else if (auto data_split = std::dynamic_pointer_cast(split)) {
         if (HasIndexScoreField(raw_read_schema_)) {
             return Status::Invalid(
                 "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit");
         }
-        return InnerCreateReader(data_split, /*row_ranges=*/std::nullopt);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr inner_reader,
+                               InnerCreateReader(data_split, /*row_ranges=*/std::nullopt));
+        return WrapWithBlobViewResolverIfNeeded(data_split, std::move(inner_reader));
     }
     return Status::Invalid("Invalid Split, cannot cast to IndexedSplit or DataSplit");
 }
 
+Result> DataEvolutionSplitRead::WrapWithBlobViewResolverIfNeeded(
+    const std::shared_ptr& data_split,
+    std::unique_ptr&& inner_reader) const {
+    if (!options_.BlobViewResolveEnabled()) {
+        // preserve serialized BlobViewStruct bytes, e.g. for forwarding blob view values to
+        // another blob-view table
+        return std::move(inner_reader);
+    }
+    std::vector read_blob_view_fields = HasBlobViewField(options_, raw_read_schema_);
+    if (read_blob_view_fields.empty()) {
+        return std::move(inner_reader);
+    }
+    std::optional warehouse_path = options_.GetBlobViewUpstreamWarehouse();
+    if (!warehouse_path) {
+        return Status::Invalid(
+            "invalid config for blob view, supposed to set BLOB_VIEW_UPSTREAM_WAREHOUSE");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr pre_reader,
+                           CreateBlobViewReader(data_split, read_blob_view_fields));
+    PAIMON_ASSIGN_OR_RAISE(std::unordered_set blob_view_structs,
+                           ExtractBlobViewStructs(pre_reader.get()));
+    auto catalog_context = std::make_shared(
+        warehouse_path.value(), options_.ToMap(), options_.GetFileSystem());
+    // use global thread number
+    uint32_t cpu_count = std::thread::hardware_concurrency();
+    uint32_t thread_num = cpu_count > 0 ? cpu_count : 1;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateDefaultExecutor(thread_num));
+    PAIMON_ASSIGN_OR_RAISE(
+        BlobViewResolver resolver,
+        BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_, executor));
+    return std::make_unique(
+        std::move(inner_reader), std::move(read_blob_view_fields), std::move(resolver), pool_);
+}
+
+Result> DataEvolutionSplitRead::CreateBlobViewReader(
+    const std::shared_ptr& data_split,
+    const std::vector& read_blob_view_fields) const {
+    auto split_impl = dynamic_cast(data_split.get());
+    if (split_impl == nullptr) {
+        return Status::Invalid("unexpected error, split cast to impl failed");
+    }
+    assert(raw_read_schema_->num_fields() > 0);
+
+    arrow::FieldVector blob_view_arrow_fields;
+    blob_view_arrow_fields.reserve(read_blob_view_fields.size());
+    for (const auto& field_name : read_blob_view_fields) {
+        auto field = raw_read_schema_->GetFieldByName(field_name);
+        if (field == nullptr) {
+            return Status::Invalid(
+                fmt::format("Blob view field {} not found in read schema.", field_name));
+        }
+        blob_view_arrow_fields.push_back(std::move(field));
+    }
+    auto blob_view_schema = arrow::schema(std::move(blob_view_arrow_fields));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr data_file_path_factory,
+        path_factory_->CreateDataFilePathFactory(split_impl->Partition(), split_impl->Bucket()));
+
+    // skip blob files: they only contain blob payloads, not the blob-view columns.
+    std::vector> data_files;
+    data_files.reserve(split_impl->DataFiles().size());
+    for (const auto& file : split_impl->DataFiles()) {
+        if (!BlobUtils::IsBlobFile(file->file_name)) {
+            data_files.push_back(file);
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector> raw_file_readers,
+        CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema,
+                             /*predicate=*/nullptr, /*dv_factory=*/nullptr,
+                             /*row_ranges=*/std::nullopt, data_file_path_factory,
+                             /*extra_format_options=*/{}));
+
+    auto batch_readers =
+        ObjectUtils::MoveVector>(std::move(raw_file_readers));
+    return std::make_unique(std::move(batch_readers), pool_);
+}
+
+Result> DataEvolutionSplitRead::ExtractBlobViewStructs(
+    BatchReader* reader) {
+    if (reader == nullptr) {
+        return Status::Invalid("invalid reader in ExtractBlobViewStructs, reader is nullptr");
+    }
+    std::unordered_set blob_view_structs;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
+        if (BatchReader::IsEofBatch(batch)) {
+            break;
+        }
+        auto& [c_array, c_schema] = batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+        auto struct_array = std::dynamic_pointer_cast(arrow_array);
+        if (struct_array == nullptr) {
+            return Status::Invalid(
+                "invalid array in ExtractBlobViewStructs, batch array is not a StructArray.");
+        }
+
+        for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) {
+            auto binary_array =
+                std::dynamic_pointer_cast(struct_array->field(field_idx));
+            if (binary_array == nullptr) {
+                return Status::Invalid(
+                    "invalid array in ExtractBlobViewStructs, blob view column is not a "
+                    "LargeBinaryArray.");
+            }
+            for (int64_t row = 0; row < binary_array->length(); ++row) {
+                if (binary_array->IsNull(row)) {
+                    continue;
+                }
+                std::string_view bytes = binary_array->GetView(row);
+                PAIMON_ASSIGN_OR_RAISE(
+                    bool is_view, BlobViewStruct::IsBlobViewStruct(bytes.data(), bytes.size()));
+                if (!is_view) {
+                    return Status::Invalid(
+                        "blob-view-field requires blob field value to be a serialized "
+                        "BlobViewStruct.");
+                }
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr view_struct,
+                                       BlobViewStruct::Deserialize(bytes.data(), bytes.size()));
+                blob_view_structs.insert(*view_struct);
+            }
+        }
+    }
+    return blob_view_structs;
+}
+
 Result> DataEvolutionSplitRead::InnerCreateReader(
     const std::shared_ptr& data_split,
     const std::optional>& row_ranges) const {
@@ -160,7 +309,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader(
                 std::vector> raw_file_readers,
                 CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_,
                                      /*predicate=*/nullptr,
-                                     /*dv_factory=*/nullptr, row_ranges, data_file_path_factory));
+                                     /*dv_factory=*/nullptr, row_ranges, data_file_path_factory,
+                                     /*extra_format_options=*/{}));
             assert(raw_file_readers.size() == 1);
             sub_readers.push_back(std::move(raw_file_readers[0]));
         } else {
@@ -176,6 +326,7 @@ Result> DataEvolutionSplitRead::InnerCreateReader(
         ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), context_->GetPredicate()));
     return std::make_unique(std::move(batch_reader), pool_);
 }
+
 Result> DataEvolutionSplitRead::ApplyIndexAndDvReaderIfNeeded(
     std::unique_ptr&& file_reader, const std::shared_ptr& file,
     const std::shared_ptr& data_schema,
@@ -335,18 +486,30 @@ Result> DataEvolutionSplitRead::CreateU
         if (!read_fields_in_file.empty()) {
             // create new FieldMappingReader for read partial fields
             auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file);
-            PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers,
-                                   CreateRawFileReaders(partition, bunch->Files(), file_read_schema,
-                                                        /*predicate=*/nullptr, /*dv_factory=*/{},
-                                                        row_ranges, data_file_path_factory));
-            if (file_readers.size() == 1) {
-                file_batch_readers[file_idx] = std::move(file_readers[0]);
+            auto blob_bunch = std::dynamic_pointer_cast(bunch);
+            if (blob_bunch && !blob_bunch->SequentialReadOptimize()) {
+                // blob files span multiple max sequence number layers: placeholder entries of
+                // newer layers must fall back row by row to older layers
+                PAIMON_ASSIGN_OR_RAISE(
+                    file_batch_readers[file_idx],
+                    CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema,
+                                             row_ranges, data_file_path_factory));
             } else {
-                auto raw_readers =
-                    ObjectUtils::MoveVector>(std::move(file_readers));
-                // Concat multiple blob files that map to the same data file.
-                file_batch_readers[file_idx] =
-                    std::make_unique(std::move(raw_readers), pool_);
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::vector> file_readers,
+                    CreateRawFileReaders(partition, bunch->Files(), file_read_schema,
+                                         /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
+                                         data_file_path_factory,
+                                         /*extra_format_options=*/{}));
+                if (file_readers.size() == 1) {
+                    file_batch_readers[file_idx] = std::move(file_readers[0]);
+                } else {
+                    auto raw_readers = ObjectUtils::MoveVector>(
+                        std::move(file_readers));
+                    // Concat multiple blob files that map to the same data file.
+                    file_batch_readers[file_idx] =
+                        std::make_unique(std::move(raw_readers), pool_);
+                }
             }
         }
     }
@@ -356,6 +519,98 @@ Result> DataEvolutionSplitRead::CreateU
                                            field_offsets, pool_);
 }
 
+namespace {
+/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a
+/// selection, otherwise its intersection with the (possibly overlapping) selected ranges.
+std::vector SelectedRangesInRange(int64_t from, int64_t to,
+                                         const std::optional>& row_ranges) {
+    if (!row_ranges) {
+        return {Range(from, to)};
+    }
+    std::vector selected;
+    Range gap_range(from, to);
+    for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) {
+        std::optional intersection = Range::Intersection(gap_range, range);
+        if (intersection) {
+            selected.push_back(*intersection);
+        }
+    }
+    return selected;
+}
+}  // namespace
+
+Result> DataEvolutionSplitRead::CreateBlobFallbackReader(
+    const BinaryRow& partition, const std::vector>& files,
+    const std::shared_ptr& file_read_schema,
+    const std::optional>& row_ranges,
+    const std::shared_ptr& data_file_path_factory) const {
+    int64_t union_first_row_id = std::numeric_limits::max();
+    int64_t union_last_row_id = std::numeric_limits::min();
+    using FileWithFirstRowId = std::pair>;
+    std::map, std::greater<>> sequence_groups;
+    for (const auto& file : files) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
+        union_first_row_id = std::min(union_first_row_id, first_row_id);
+        union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1);
+        sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file);
+    }
+    // the blob format reader must emit placeholder sentinels instead of failing on them
+    const std::map blob_format_options = {
+        {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
+    std::vector> groups;
+    groups.reserve(sequence_groups.size());
+    for (auto& [max_sequence_number, group_files] : sequence_groups) {
+        std::stable_sort(group_files.begin(), group_files.end(),
+                         [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) {
+                             return f1.first < f2.first;
+                         });
+        std::vector segments;
+        // pad row ids this layer does not cover with placeholder gaps, so that every layer spans
+        // the same union range and the groups can be stepped in lockstep
+        int64_t next_row_id = union_first_row_id;
+        for (const auto& [first_row_id, file] : group_files) {
+            if (first_row_id < next_row_id) {
+                return Status::Invalid(fmt::format(
+                    "Blob files with the same max sequence number should not have overlapping "
+                    "row id ranges: file {} (max sequence number {}) starts at row id {} before "
+                    "the previous file's end {}",
+                    file->file_name, max_sequence_number, first_row_id, next_row_id));
+            }
+            if (first_row_id > next_row_id) {
+                std::vector gap_selected_ranges =
+                    SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges);
+                if (!gap_selected_ranges.empty()) {
+                    segments.push_back(
+                        BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
+                }
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector> file_readers,
+                CreateRawFileReaders(partition, {file}, file_read_schema,
+                                     /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
+                                     data_file_path_factory, blob_format_options));
+            if (file_readers.size() != 1) {
+                return Status::Invalid("Unexpected: blob fallback file reader was skipped.");
+            }
+            segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}});
+            next_row_id = first_row_id + file->row_count;
+        }
+        if (next_row_id <= union_last_row_id) {
+            std::vector gap_selected_ranges =
+                SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges);
+            if (!gap_selected_ranges.empty()) {
+                segments.push_back(
+                    BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
+            }
+        }
+        groups.push_back(std::move(segments));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader,
+                           BlobFallbackBatchReader::Create(std::move(groups), file_read_schema,
+                                                           options_.GetReadBatchSize(), pool_));
+    return std::move(fallback_reader);
+}
+
 Result DataEvolutionSplitRead::Match(const std::shared_ptr& split,
                                            bool force_keep_delete) const {
     return true;
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index 0129b007..9c45b32f 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -20,11 +20,15 @@
 
 #include 
 #include 
+#include 
+#include 
 #include 
 #include 
 #include 
+#include 
 #include 
 
+#include "paimon/common/data/blob_view_struct.h"
 #include "paimon/common/reader/data_evolution_file_reader.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/operation/abstract_split_read.h"
@@ -53,9 +57,12 @@ struct DeletionFile;
 /// otherwise, it must be present in the read path.
 ///
 /// Readers Overview: (ConcatBatchReader across
-/// splits)->(CompleteIndexScoreBatchReader)->CompleteRowKindBatchReader->(PredicateBatchReader)
-/// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files)
-/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)
+/// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)->
+/// CompleteRowKindBatchReader->(PredicateBatchReader)
+/// ->ConcatBatchReader across files->DataEvolutionFileReader
+/// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers)
+/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader)
+/// ->(MapSharedShreddingFileReader)
 /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 ///
 ///
@@ -105,26 +112,43 @@ class DataEvolutionSplitRead : public AbstractSplitRead {
         std::vector> data_files_;
     };
 
+    /// All blob files of one blob field in a merge group. Unlike data files, every file is kept
+    /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers
+    /// of a data-evolution partial update, where newer layers record untouched rows as
+    /// placeholder entries and reading falls back row by row to older layers.
+    /// Files must be added ordered by first row id ascending (then max sequence number
+    /// descending), as produced by MergeRangesAndSort.
     class BlobBunch : public FieldBunch {
      public:
         explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection)
             : expected_row_count_(expected_row_count),
               has_row_ids_selection_(has_row_ids_selection) {}
-        int64_t RowCount() const override {
-            return row_count_;
-        }
+        /// Number of distinct row ids covered by the added files. Without a row-ids selection
+        /// the covered range is contiguous (enforced by Add) and matches the data files.
+        int64_t RowCount() const override;
         const std::vector>& Files() const override {
             return files_;
         }
         Status Add(const std::shared_ptr& file);
+        /// True when every file shares one max sequence number, so the files are read
+        /// sequentially without the fallback merge. A lone layer is expected to hold no
+        /// placeholder entries; if one does (a user value equal to the placeholder sentinel
+        /// written by a blob-only first write), the strict blob reader rejects the read, since
+        /// no older layer exists to resolve it.
+        bool SequentialReadOptimize() const {
+            return sequence_group_end_.size() <= 1;
+        }
 
      private:
         int64_t expected_row_count_ = -1;
-        int64_t latest_first_row_id_ = -1;
-        int64_t expected_next_first_row_id_ = -1;
-        int64_t latest_max_sequence_number_ = -1;
-        int64_t row_count_ = 0;
         bool has_row_ids_selection_ = false;
+        int64_t union_first_row_id_ = std::numeric_limits::max();
+        /// Exclusive end of the union row id range covered so far.
+        int64_t union_end_row_id_ = std::numeric_limits::min();
+        /// Per max sequence number: exclusive end of the last added range, to reject
+        /// overlapping files within one layer.
+        std::map sequence_group_end_;
+        std::vector ranges_;
         std::vector> files_;
     };
 
@@ -143,12 +167,35 @@ class DataEvolutionSplitRead : public AbstractSplitRead {
 
     static bool HasIndexScoreField(const std::shared_ptr& read_schema);
 
+    static std::vector HasBlobViewField(
+        const CoreOptions& options, const std::shared_ptr& read_schema);
+
+    static Result> ExtractBlobViewStructs(BatchReader* reader);
+
+    Result> CreateBlobViewReader(
+        const std::shared_ptr& data_split,
+        const std::vector& read_blob_view_fields) const;
+
+    Result> WrapWithBlobViewResolverIfNeeded(
+        const std::shared_ptr& data_split,
+        std::unique_ptr&& inner_reader) const;
+
  private:
     Result> CreateUnionReader(
         const BinaryRow& partition,
         const std::vector>& need_merge_files,
         const std::optional>& row_ranges,
         const std::shared_ptr& data_file_path_factory) const;
+
+    /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence
+    /// number layers: groups the files by max sequence number, pads uncovered row id ranges of
+    /// each layer with placeholder gap segments, and resolves each row to the newest
+    /// non-placeholder layer. See BlobFallbackBatchReader.
+    Result> CreateBlobFallbackReader(
+        const BinaryRow& partition, const std::vector>& files,
+        const std::shared_ptr& file_read_schema,
+        const std::optional>& row_ranges,
+        const std::shared_ptr& data_file_path_factory) const;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp
index d44c3992..597d668d 100644
--- a/src/paimon/core/operation/data_evolution_split_read_test.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp
@@ -147,72 +147,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) {
 TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
-                       /*max_sequence_number=*/1,
+                       /*max_sequence_number=*/3,
                        /*write_cols=*/std::optional>({"blob_col"}));
-    auto blob_tail =
-        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50,
+    auto blob_full_tail =
+        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
+    auto blob_short_tail =
+        CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50,
+                       /*max_sequence_number=*/1,
+                       /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail),
-                        "Blob file with same first row id should have decreasing sequence number.");
+    // Files with the same first row id and lower sequence numbers are older layers of a
+    // partial update; they are kept for the row-level placeholder fallback, whether they
+    // cover the same range or only a shorter prefix of it.
+    ASSERT_OK(blob_bunch->Add(blob_full_tail));
+    ASSERT_OK(blob_bunch->Add(blob_short_tail));
+
+    ASSERT_EQ(blob_bunch->Files().size(), 3);
+    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail);
+    ASSERT_EQ(blob_bunch->RowCount(), 100);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
-        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50,
+        CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
                        /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // Adding file with same firstRowId and lower sequence number should be ignored
+    // Overlapping layers with different sequence numbers are kept for the fallback.
     ASSERT_OK(blob_bunch->Add(blob_tail));
 
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->RowCount(), 200);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
-                       /*max_sequence_number=*/2,
+                       /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
         CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
-                       /*max_sequence_number=*/1,
+                       /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // Adding file with overlapping row id and lower sequence number should be ignored
     ASSERT_OK(blob_bunch->Add(blob_tail));
 
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->RowCount(), 200);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
         CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
-                       /*max_sequence_number=*/2,
+                       /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    ASSERT_NOK_WITH_MSG(
-        blob_bunch->Add(blob_tail),
-        "Blob file with overlapping row id should have decreasing sequence number.");
+    // Files sharing a max sequence number form one layer and must not overlap.
+    ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail),
+                        "Blob files with the same max sequence number should not have overlapping "
+                        "row id ranges");
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) {
@@ -267,12 +283,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) {
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/true);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // blob_sub1 will not be added, for it has been skipped by row_ids in scan process.
-    // after blob_sub2 is added, blob_entry is removed
+    // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer
+    // layer and both files are kept for the row-level placeholder fallback.
     ASSERT_OK(blob_bunch->Add(blob_sub2));
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_sub2);
-    ASSERT_EQ(blob_bunch->RowCount(), 5);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_sub2);
+    ASSERT_EQ(blob_bunch->RowCount(), 10);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) {
@@ -293,13 +311,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) {
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/true);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // blob_sub1 will not be added, for it has been skipped by row_ids in scan process.
-    // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be
-    // skipped.
+    // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older
+    // layer and both files are kept for the row-level placeholder fallback.
     ASSERT_OK(blob_bunch->Add(blob_sub2));
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_sub2);
     ASSERT_EQ(blob_bunch->RowCount(), 10);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) {
@@ -422,15 +441,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) {
     std::vector> batch = batches[0];
     ASSERT_EQ(batch.size(), 10);
     ASSERT_EQ(batch[0], data);
-    ASSERT_EQ(batch[1], blob_entry5);  // pick
-    ASSERT_EQ(batch[2], blob_entry2);  // skip
-    ASSERT_EQ(batch[3], blob_entry1);  // skip
-    ASSERT_EQ(batch[4], blob_entry9);  // pick
-    ASSERT_EQ(batch[5], blob_entry6);  // skip
-    ASSERT_EQ(batch[6], blob_entry3);  // skip
-    ASSERT_EQ(batch[7], blob_entry7);  // pick
-    ASSERT_EQ(batch[8], blob_entry4);  // skip
-    ASSERT_EQ(batch[9], blob_entry8);  // pick
+    ASSERT_EQ(batch[1], blob_entry5);
+    ASSERT_EQ(batch[2], blob_entry2);
+    ASSERT_EQ(batch[3], blob_entry1);
+    ASSERT_EQ(batch[4], blob_entry9);
+    ASSERT_EQ(batch[5], blob_entry6);
+    ASSERT_EQ(batch[6], blob_entry3);
+    ASSERT_EQ(batch[7], blob_entry7);
+    ASSERT_EQ(batch[8], blob_entry4);
+    ASSERT_EQ(batch[9], blob_entry8);
 
     auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result {
         return 0;
@@ -442,12 +461,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) {
     ASSERT_EQ(bunch.size(), 2);
     auto blob_bunch = std::dynamic_pointer_cast(bunch[1]);
 
-    ASSERT_EQ(blob_bunch->Files().size(), 4);
+    // every sequence layer is kept for the row-level placeholder fallback
+    ASSERT_EQ(blob_bunch->Files().size(), 9);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry5);
-    ASSERT_EQ(blob_bunch->Files()[1], blob_entry9);
-    ASSERT_EQ(blob_bunch->Files()[2], blob_entry7);
-    ASSERT_EQ(blob_bunch->Files()[3], blob_entry8);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_entry2);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_entry1);
+    ASSERT_EQ(blob_bunch->Files()[3], blob_entry9);
+    ASSERT_EQ(blob_bunch->Files()[4], blob_entry6);
+    ASSERT_EQ(blob_bunch->Files()[5], blob_entry3);
+    ASSERT_EQ(blob_bunch->Files()[6], blob_entry7);
+    ASSERT_EQ(blob_bunch->Files()[7], blob_entry4);
+    ASSERT_EQ(blob_bunch->Files()[8], blob_entry8);
     ASSERT_EQ(blob_bunch->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) {
@@ -563,20 +589,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) {
 
     ASSERT_EQ(bunch.size(), 3);
     auto blob_bunch = std::dynamic_pointer_cast(bunch[1]);
-    ASSERT_EQ(blob_bunch->Files().size(), 4);
+    // every sequence layer is kept for the row-level placeholder fallback
+    ASSERT_EQ(blob_bunch->Files().size(), 9);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry5);
-    ASSERT_EQ(blob_bunch->Files()[1], blob_entry9);
-    ASSERT_EQ(blob_bunch->Files()[2], blob_entry7);
-    ASSERT_EQ(blob_bunch->Files()[3], blob_entry8);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_entry2);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_entry1);
+    ASSERT_EQ(blob_bunch->Files()[3], blob_entry9);
+    ASSERT_EQ(blob_bunch->Files()[4], blob_entry6);
+    ASSERT_EQ(blob_bunch->Files()[5], blob_entry3);
+    ASSERT_EQ(blob_bunch->Files()[6], blob_entry7);
+    ASSERT_EQ(blob_bunch->Files()[7], blob_entry4);
+    ASSERT_EQ(blob_bunch->Files()[8], blob_entry8);
     ASSERT_EQ(blob_bunch->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 
     auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]);
-    ASSERT_EQ(blob_bunch2->Files().size(), 4);
+    ASSERT_EQ(blob_bunch2->Files().size(), 9);
     ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15);
-    ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19);
-    ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17);
-    ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18);
+    ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12);
+    ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11);
+    ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19);
+    ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16);
+    ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13);
+    ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17);
+    ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14);
+    ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18);
     ASSERT_EQ(blob_bunch2->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch2->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) {
diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp
index 5c40c442..d6b965ba 100644
--- a/src/paimon/core/operation/expire_snapshots_test.cpp
+++ b/src/paimon/core/operation/expire_snapshots_test.cpp
@@ -75,9 +75,10 @@ class ExpireSnapshotsTest : public testing::Test {
         test_data_path_ = "tmp";
         path_factory_ = CreateFactory(test_data_path_);
 
-        ASSERT_OK_AND_ASSIGN(manifest_list_, ManifestList::Create(fs_, options.GetManifestFormat(),
-                                                                  options.GetManifestCompression(),
-                                                                  path_factory_, mem_pool_));
+        ASSERT_OK_AND_ASSIGN(
+            manifest_list_,
+            ManifestList::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(),
+                                 path_factory_, options.GetCache(), mem_pool_));
 
         ASSERT_OK_AND_ASSIGN(
             manifest_file_,
diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp
index 8cc4ba9a..b2ba17fa 100644
--- a/src/paimon/core/operation/file_store_commit.cpp
+++ b/src/paimon/core/operation/file_store_commit.cpp
@@ -30,8 +30,11 @@
 #include "paimon/core/manifest/index_manifest_file.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/operation/append_only_file_store_scan.h"
 #include "paimon/core/operation/expire_snapshots.h"
 #include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/key_value_file_store_scan.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/utils/field_mapping.h"
@@ -47,6 +50,50 @@ class Schema;
 
 namespace paimon {
 
+namespace {
+
+CommitScanner::ScanSupplier CreateAppendScanSupplier(
+    const std::shared_ptr& snapshot_manager,
+    const std::shared_ptr& schema_manager,
+    const std::shared_ptr& manifest_list,
+    const std::shared_ptr& manifest_file,
+    const std::shared_ptr& table_schema,
+    const std::shared_ptr& arrow_schema, const CoreOptions& options,
+    const std::shared_ptr& executor, const std::shared_ptr& pool) {
+    return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
+            arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter)
+               -> Result> {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr scan,
+            AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
+                                            manifest_file, table_schema, arrow_schema, scan_filter,
+                                            options, executor, pool));
+        return std::unique_ptr(std::move(scan));
+    };
+}
+
+CommitScanner::ScanSupplier CreatePkScanSupplier(
+    const std::shared_ptr& snapshot_manager,
+    const std::shared_ptr& schema_manager,
+    const std::shared_ptr& manifest_list,
+    const std::shared_ptr& manifest_file,
+    const std::shared_ptr& table_schema,
+    const std::shared_ptr& arrow_schema, const CoreOptions& options,
+    const std::shared_ptr& executor, const std::shared_ptr& pool) {
+    return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
+            arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter)
+               -> Result> {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr scan,
+            KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
+                                          manifest_file, table_schema, arrow_schema, scan_filter,
+                                          options, executor, pool));
+        return std::unique_ptr(std::move(scan));
+    };
+}
+
+}  // namespace
+
 Result> FileStoreCommit::Create(
     std::unique_ptr ctx) {
     if (ctx == nullptr) {
@@ -69,10 +116,6 @@ Result> FileStoreCommit::Create(
         return Status::Invalid("not found latest schema");
     }
     const auto& schema = table_schema.value();
-    if (!schema->PrimaryKeys().empty() &&
-        ctx->GetOptions().find("enable-pk-commit-in-inte-test") == ctx->GetOptions().end()) {
-        return Status::NotImplemented("not support pk table commit yet");
-    }
     auto opts = schema->Options();
     for (const auto& [key, value] : ctx->GetOptions()) {
         opts[key] = value;
@@ -83,8 +126,11 @@ Result> FileStoreCommit::Create(
                            CoreOptions::FromMap(opts, ctx->GetSpecificFileSystem()));
     assert(options.GetFileSystem());
     assert(options.GetFileFormat());
+    PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options));
+
     PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path));
-    if (is_object_store && opts.find("enable-object-store-commit-in-inte-test") == opts.end()) {
+    if (is_object_store && !ctx->UseRESTCatalogCommit() &&
+        opts.find("enable-object-store-commit-in-inte-test") == opts.end()) {
         return Status::NotImplemented(
             "commit operation does not support object store file system for now");
     }
@@ -109,7 +155,8 @@ Result> FileStoreCommit::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_list,
         ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(),
-                             options.GetManifestCompression(), path_factory, ctx->GetMemoryPool()));
+                             options.GetManifestCompression(), path_factory, options.GetCache(),
+                             ctx->GetMemoryPool()));
 
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr partition_schema,
@@ -130,11 +177,23 @@ Result> FileStoreCommit::Create(
         snapshot_manager, path_factory, manifest_list, manifest_file, options.GetFileSystem(),
         options.GetExpireConfig(), ctx->GetExecutor());
 
+    CommitScanner::ScanSupplier scan_supplier;
+    if (table_schema.value()->PrimaryKeys().empty()) {
+        scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list,
+                                                 manifest_file, table_schema.value(), arrow_schema,
+                                                 options, ctx->GetExecutor(), ctx->GetMemoryPool());
+    } else {
+        scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list,
+                                             manifest_file, table_schema.value(), arrow_schema,
+                                             options, ctx->GetExecutor(), ctx->GetMemoryPool());
+    }
+
     return std::make_unique(
         ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(),
         options, path_factory, std::move(partition_computer), snapshot_manager,
-        ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), table_schema.value(), manifest_file,
-        manifest_list, index_manifest_file, expire_snapshots, schema_manager);
+        ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), ctx->AppendCommitCheckConflict(),
+        table_schema.value(), manifest_file, manifest_list, index_manifest_file, expire_snapshots,
+        schema_manager, std::move(scan_supplier));
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp
index 06eeeae5..6342f9b3 100644
--- a/src/paimon/core/operation/file_store_commit_impl.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl.cpp
@@ -21,10 +21,13 @@
 
 #include 
 #include 
+#include 
 #include 
+#include 
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "fmt/format.h"
@@ -35,8 +38,11 @@
 #include "paimon/common/executor/future.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/binary_row_partition_computer.h"
 #include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/catalog/catalog_snapshot_commit.h"
 #include "paimon/core/catalog/renaming_snapshot_commit.h"
@@ -57,14 +63,19 @@
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/manifest/partition_entry.h"
-#include "paimon/core/operation/append_only_file_store_scan.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h"
+#include "paimon/core/operation/commit/conflict_detection.h"
+#include "paimon/core/operation/commit/row_tracking_commit_utils.h"
+#include "paimon/core/operation/commit/sequence_snapshot_properties.h"
 #include "paimon/core/operation/expire_snapshots.h"
-#include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/operation/manifest_file_merger.h"
 #include "paimon/core/operation/metrics/commit_metrics.h"
+#include "paimon/core/operation/metrics/commit_stats.h"
 #include "paimon/core/partition/partition_statistics.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/bucket_mode.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
 #include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/file_store_path_factory.h"
@@ -72,12 +83,58 @@
 #include "paimon/fs/file_system.h"
 #include "paimon/logging.h"
 #include "paimon/metrics.h"
-#include "paimon/scan_context.h"
 
 namespace paimon {
 class Executor;
 class MemoryPool;
 
+namespace {
+
+constexpr const char* kCommitStrictModeLastSafeSnapshot = "commit.strict-mode.last-safe-snapshot";
+constexpr const char* kManifestDeleteFileDropStats = "manifest.delete-file-drop-stats";
+constexpr const char* kSequenceSnapshotOrdering = "sequence.snapshot-ordering";
+constexpr const char* kPkClusteringOverride = "pk-clustering-override";
+
+bool MatchPartitionSpec(const std::map& partition,
+                        const std::map& partition_spec) {
+    for (const auto& [key, value] : partition_spec) {
+        auto iter = partition.find(key);
+        if (iter == partition.end() || iter->second != value) {
+            return false;
+        }
+    }
+    return true;
+}
+
+}  // namespace
+
+Status FileStoreCommitImpl::ValidateCommitOptions(const CoreOptions& options) {
+    const auto& raw_options = options.ToMap();
+    std::vector unsupported_options;
+
+    if (raw_options.find(kCommitStrictModeLastSafeSnapshot) != raw_options.end()) {
+        unsupported_options.emplace_back(kCommitStrictModeLastSafeSnapshot);
+    }
+    if (raw_options.find(kManifestDeleteFileDropStats) != raw_options.end()) {
+        unsupported_options.emplace_back(kManifestDeleteFileDropStats);
+    }
+    if (raw_options.find(kSequenceSnapshotOrdering) != raw_options.end()) {
+        unsupported_options.emplace_back(kSequenceSnapshotOrdering);
+    }
+    if (raw_options.find(kPkClusteringOverride) != raw_options.end()) {
+        unsupported_options.emplace_back(kPkClusteringOverride);
+    }
+
+    if (!unsupported_options.empty()) {
+        return Status::Invalid(fmt::format(
+            "These options are not supported by C++ commit path: {}. "
+            "Please use Java commit, or remove these options before creating FileStoreCommit.",
+            fmt::join(unsupported_options, ", ")));
+    }
+
+    return Status::OK();
+}
+
 FileStoreCommitImpl::FileStoreCommitImpl(
     const std::shared_ptr& pool, const std::shared_ptr& executor,
     const std::shared_ptr& schema, const std::string& root_path,
@@ -85,16 +142,18 @@ FileStoreCommitImpl::FileStoreCommitImpl(
     const std::shared_ptr& path_factory,
     std::unique_ptr partition_computer,
     const std::shared_ptr& snapshot_manager, bool ignore_empty_commit,
-    bool use_rest_catalog_commit, const std::shared_ptr& table_schema,
+    bool use_rest_catalog_commit, bool append_commit_check_conflict,
+    const std::shared_ptr& table_schema,
     const std::shared_ptr& manifest_file,
     const std::shared_ptr& manifest_list,
     const std::shared_ptr& index_manifest_file,
     const std::shared_ptr& expire_snapshots,
-    const std::shared_ptr& schema_manager)
+    const std::shared_ptr& schema_manager, CommitScanner::ScanSupplier scan_supplier)
     : memory_pool_(pool),
       executor_(executor),
       schema_(schema),
       root_path_(root_path),
+      table_name_(PathUtil::GetName(root_path)),
       commit_user_(commit_user),
       options_(options),
       path_factory_(path_factory),
@@ -102,8 +161,17 @@ FileStoreCommitImpl::FileStoreCommitImpl(
       partition_computer_(std::move(partition_computer)),
       snapshot_manager_(snapshot_manager),
       ignore_empty_commit_(ignore_empty_commit),
+      append_commit_check_conflict_(append_commit_check_conflict),
+      retry_waiter_(options.GetCommitMinRetryWait(), options.GetCommitMaxRetryWait()),
       num_bucket_(options.GetBucket()),
+      bucket_mode_(ResolveBucketMode(options.GetBucket(), table_schema)),
       table_schema_(table_schema),
+      commit_scanner_(std::make_shared(
+          snapshot_manager, schema_manager, manifest_list, manifest_file, index_manifest_file,
+          table_schema, schema, options, executor, pool, partition_computer_.get(),
+          std::move(scan_supplier))),
+      conflict_detection_(table_schema, options, snapshot_manager_, manifest_list, manifest_file,
+                          commit_scanner_, commit_user, table_name_, path_factory),
       manifest_file_(manifest_file),
       manifest_list_(manifest_list),
       index_manifest_file_(index_manifest_file),
@@ -131,7 +199,182 @@ Status FileStoreCommitImpl::DropPartition(
     }
     std::string log_msg = fmt::format("Ready to drop partitions {}", partitions);
     PAIMON_LOG_DEBUG(logger_, "%s", log_msg.c_str());
-    return TryOverwrite(partitions, {}, commit_identifier, std::nullopt);
+    PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t attempt,
+                           TryOverwrite(partitions, /*changes=*/{}, /*index_entries=*/{},
+                                        commit_identifier, std::nullopt, /*properties=*/{}));
+    return Status::OK();
+}
+
+Status FileStoreCommitImpl::TruncateTable(int64_t commit_identifier) {
+    // An empty partition list means "all partitions", so this overwrites the whole table with
+    // no new files, effectively truncating it. Mirrors Java tryOverwritePartition(null, ...).
+    PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t attempt,
+                           TryOverwrite(/*partition=*/{}, /*changes=*/{}, /*index_entries=*/{},
+                                        commit_identifier, std::nullopt, /*properties=*/{}));
+    return Status::OK();
+}
+
+Status FileStoreCommitImpl::Abort(
+    const std::vector>& commit_messages) {
+    for (const auto& message : commit_messages) {
+        auto* msg = dynamic_cast(message.get());
+        if (msg == nullptr) {
+            return Status::Invalid("fail to cast commit message to impl");
+        }
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr data_file_path_factory,
+            path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr index_file_path_factory,
+            path_factory_->CreateIndexFileFactory(msg->Partition(), msg->Bucket()));
+
+        const DataIncrement& new_files_increment = msg->GetNewFilesIncrement();
+        const CompactIncrement& compact_increment = msg->GetCompactIncrement();
+
+        std::vector> data_files_to_delete;
+        auto append_data_files =
+            [&data_files_to_delete](const std::vector>& files) {
+                data_files_to_delete.insert(data_files_to_delete.end(), files.begin(), files.end());
+            };
+        append_data_files(new_files_increment.NewFiles());
+        append_data_files(new_files_increment.ChangelogFiles());
+        append_data_files(compact_increment.CompactAfter());
+        append_data_files(compact_increment.ChangelogFiles());
+        for (const auto& file : data_files_to_delete) {
+            // Best-effort cleanup: ignore delete failures, aligning with Java deleteQuietly.
+            [[maybe_unused]] Status status =
+                fs_->Delete(data_file_path_factory->ToPath(file), /*recursive=*/false);
+        }
+
+        std::vector> index_files_to_delete;
+        auto append_index_files = [&index_files_to_delete](
+                                      const std::vector>& files) {
+            index_files_to_delete.insert(index_files_to_delete.end(), files.begin(), files.end());
+        };
+        append_index_files(new_files_increment.NewIndexFiles());
+        append_index_files(compact_increment.NewIndexFiles());
+        for (const auto& file : index_files_to_delete) {
+            [[maybe_unused]] Status status =
+                fs_->Delete(index_file_path_factory->ToPath(file), /*recursive=*/false);
+        }
+    }
+    return Status::OK();
+}
+
+Result> FileStoreCommitImpl::ReadAddManifestEntries(
+    const Snapshot& snapshot) const {
+    std::vector data_manifests;
+    PAIMON_RETURN_NOT_OK(manifest_list_->ReadDataManifests(snapshot, &data_manifests));
+    std::vector unmerged_entries;
+    for (const auto& meta : data_manifests) {
+        std::vector entries;
+        PAIMON_RETURN_NOT_OK(manifest_file_->Read(
+            meta.FileName(), [](const ManifestEntry&) -> Result { return true; }, &entries));
+        unmerged_entries.insert(unmerged_entries.end(), std::make_move_iterator(entries.begin()),
+                                std::make_move_iterator(entries.end()));
+    }
+    std::vector merged_entries;
+    PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(unmerged_entries, &merged_entries));
+    std::vector add_entries;
+    for (auto& entry : merged_entries) {
+        if (entry.Kind() == FileKind::Add()) {
+            add_entries.push_back(std::move(entry));
+        }
+    }
+    return add_entries;
+}
+
+Result FileStoreCommitImpl::RollbackToAsLatest(int64_t target_snapshot_id) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional latest_opt, snapshot_manager_->LatestSnapshot());
+    if (!latest_opt) {
+        return Status::Invalid("Latest snapshot is null, can not roll back.");
+    }
+    const Snapshot& latest = latest_opt.value();
+    PAIMON_ASSIGN_OR_RAISE(Snapshot target_snapshot,
+                           snapshot_manager_->LoadSnapshot(target_snapshot_id));
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector latest_entries,
+                           ReadAddManifestEntries(latest));
+    PAIMON_ASSIGN_OR_RAISE(std::vector target_entries,
+                           ReadAddManifestEntries(target_snapshot));
+
+    std::unordered_set latest_identifiers;
+    latest_identifiers.reserve(latest_entries.size());
+    for (const auto& entry : latest_entries) {
+        latest_identifiers.insert(entry.CreateIdentifier());
+    }
+    std::unordered_set target_identifiers;
+    target_identifiers.reserve(target_entries.size());
+    for (const auto& entry : target_entries) {
+        target_identifiers.insert(entry.CreateIdentifier());
+    }
+
+    std::vector delta_files;
+    for (const auto& entry : latest_entries) {
+        if (target_identifiers.find(entry.CreateIdentifier()) == target_identifiers.end()) {
+            delta_files.emplace_back(FileKind::Delete(), entry.Partition(), entry.Bucket(),
+                                     entry.TotalBuckets(), entry.File());
+        }
+    }
+    for (const auto& entry : target_entries) {
+        if (latest_identifiers.find(entry.CreateIdentifier()) == latest_identifiers.end()) {
+            delta_files.emplace_back(FileKind::Add(), entry.Partition(), entry.Bucket(),
+                                     entry.TotalBuckets(), entry.File());
+        }
+    }
+
+    std::pair base_manifest_list;
+    std::pair delta_manifest_list;
+    PAIMON_ASSIGN_OR_RAISE(std::vector base_manifests,
+                           manifest_file_->Write(latest_entries));
+    PAIMON_ASSIGN_OR_RAISE(base_manifest_list, manifest_list_->Write(base_manifests));
+    PAIMON_ASSIGN_OR_RAISE(std::vector delta_manifests,
+                           manifest_file_->Write(delta_files));
+    PAIMON_ASSIGN_OR_RAISE(delta_manifest_list, manifest_list_->Write(delta_manifests));
+
+    // For row-tracking tables nextRowId must stay monotonic: a rollback to an older snapshot must
+    // not move it backwards, otherwise new appends would reuse row ids already assigned by the
+    // snapshots between the target and the previous latest, breaking the global uniqueness of
+    // _ROW_ID. Keep the larger of the previous latest and the target nextRowId.
+    std::optional next_row_id = std::max(latest.NextRowId(), target_snapshot.NextRowId());
+
+    int64_t delta_record_count =
+        ManifestEntry::RecordCountAdd(delta_files) - ManifestEntry::RecordCountDelete(delta_files);
+    Snapshot new_snapshot(
+        latest.Id() + 1, target_snapshot.SchemaId(), base_manifest_list.first,
+        base_manifest_list.second, delta_manifest_list.first, delta_manifest_list.second,
+        /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt,
+        target_snapshot.IndexManifest(), commit_user_,
+        /*commit_identifier=*/std::numeric_limits::max(),
+        Snapshot::CommitKind::Overwrite(), DateTimeUtils::GetCurrentUTCTimeUs() / 1000,
+        target_snapshot.TotalRecordCount(), delta_record_count,
+        /*changelog_record_count=*/std::nullopt, target_snapshot.Watermark(),
+        target_snapshot.Statistics(), target_snapshot.Properties(), next_row_id);
+
+    std::unordered_map partition_entry_map;
+    PAIMON_RETURN_NOT_OK(PartitionEntry::Merge(delta_files, &partition_entry_map));
+    std::vector delta_statistics;
+    delta_statistics.reserve(partition_entry_map.size());
+    for (const auto& [_, partition_entry] : partition_entry_map) {
+        delta_statistics.push_back(partition_entry);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(bool success, CommitSnapshotImpl(new_snapshot, delta_statistics));
+    if (success) {
+        PAIMON_LOG_INFO(logger_,
+                        "Successfully rolled back table %s to snapshot %ld as new snapshot %ld by "
+                        "user %s.",
+                        root_path_.c_str(), target_snapshot_id, new_snapshot.Id(),
+                        commit_user_.c_str());
+        last_committed_snapshot_id_ = new_snapshot.Id();
+    }
+    return success;
+}
+
+FileStoreCommit& FileStoreCommitImpl::RowIdCheckConflict(
+    std::optional row_id_check_from_snapshot) {
+    conflict_detection_.SetRowIdCheckFromSnapshot(row_id_check_from_snapshot);
+    return *this;
 }
 
 Result FileStoreCommitImpl::FilterAndCommit(
@@ -143,8 +386,15 @@ Result FileStoreCommitImpl::FilterAndCommit(
         committables.push_back(CreateManifestCommittable(identifier, msgs, watermark));
     }
 
+    std::vector> sorted_committables = committables;
+    std::sort(sorted_committables.begin(), sorted_committables.end(),
+              [](const std::shared_ptr& lhs,
+                 const std::shared_ptr& rhs) {
+                  return lhs->Identifier() < rhs->Identifier();
+              });
+
     PAIMON_ASSIGN_OR_RAISE(std::vector> retry_committables,
-                           FilterCommitted(committables));
+                           FilterCommitted(sorted_committables));
     if (!retry_committables.empty()) {
         PAIMON_RETURN_NOT_OK(CheckFilesExistence(retry_committables));
         for (const auto& committable : retry_committables) {
@@ -160,43 +410,62 @@ Status FileStoreCommitImpl::CheckFilesExistence(
     for (const auto& committable : committables) {
         for (const auto& message : committable->FileCommittables()) {
             auto msg = dynamic_cast(message.get());
-            if (msg) {
-                PAIMON_ASSIGN_OR_RAISE(
-                    std::shared_ptr data_file_path_factory,
-                    path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket()));
-                auto collect_files =
-                    [&all_paths, data_file_path_factory](
-                        const std::vector>& file_metas) {
-                        for (const auto& file_meta : file_metas) {
-                            auto paths = data_file_path_factory->CollectFiles(file_meta);
-                            all_paths.insert(all_paths.end(), paths.begin(), paths.end());
-                        }
-                    };
-                // skip compact before files, deleted index files
-                DataIncrement new_files_increment = msg->GetNewFilesIncrement();
-                collect_files(new_files_increment.NewFiles());
-                collect_files(new_files_increment.ChangelogFiles());
-                auto new_data_index_metas = new_files_increment.NewIndexFiles();
-                for (const auto& data_index_meta : new_data_index_metas) {
-                    all_paths.push_back(
-                        path_factory_->ToIndexFilePath(data_index_meta->FileName()));
+            if (msg == nullptr) {
+                return Status::Invalid("fail to cast commit message to impl");
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr data_file_path_factory,
+                path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket()));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::unique_ptr index_file_path_factory,
+                path_factory_->CreateIndexFileFactory(msg->Partition(), msg->Bucket()));
+            auto collect_files = [&all_paths, data_file_path_factory](
+                                     const std::vector>& file_metas) {
+                for (const auto& file_meta : file_metas) {
+                    auto paths = data_file_path_factory->CollectFiles(file_meta);
+                    all_paths.insert(all_paths.end(), paths.begin(), paths.end());
                 }
+            };
+            DataIncrement new_files_increment = msg->GetNewFilesIncrement();
+            collect_files(new_files_increment.NewFiles());
+            collect_files(new_files_increment.ChangelogFiles());
+            auto new_data_index_metas = new_files_increment.NewIndexFiles();
+            for (const auto& data_index_meta : new_data_index_metas) {
+                all_paths.push_back(index_file_path_factory->ToPath(data_index_meta));
+            }
 
-                CompactIncrement compact_increment = msg->GetCompactIncrement();
-                collect_files(compact_increment.CompactBefore());
-                collect_files(compact_increment.CompactAfter());
-                auto new_compact_index_metas = compact_increment.NewIndexFiles();
-                for (const auto& compact_index_meta : new_compact_index_metas) {
-                    all_paths.push_back(
-                        path_factory_->ToIndexFilePath(compact_index_meta->FileName()));
-                }
-            } else {
-                return Status::Invalid("fail to cast commit message to impl");
+            CompactIncrement compact_increment = msg->GetCompactIncrement();
+            collect_files(compact_increment.CompactAfter());
+            auto new_compact_index_metas = compact_increment.NewIndexFiles();
+            for (const auto& compact_index_meta : new_compact_index_metas) {
+                all_paths.push_back(index_file_path_factory->ToPath(compact_index_meta));
             }
+
+            // skip compact before files, deleted index files
         }
     }
-    std::vector>>> file_exists_futures;
+
+    // Resolve compacted changelog files to their real file paths
+    std::vector resolved_paths;
+    resolved_paths.reserve(all_paths.size());
     for (const auto& path : all_paths) {
+        resolved_paths.push_back(CompactedChangelogPathResolver::Resolve(path));
+    }
+
+    // Deduplicate paths as multiple compacted changelog references may resolve to the same
+    // physical file
+    std::unordered_set deduplicated_paths_set;
+    deduplicated_paths_set.reserve(resolved_paths.size());
+    std::vector deduplicated_paths;
+    deduplicated_paths.reserve(resolved_paths.size());
+    for (const auto& path : resolved_paths) {
+        if (deduplicated_paths_set.insert(path).second) {
+            deduplicated_paths.push_back(path);
+        }
+    }
+
+    std::vector>>> file_exists_futures;
+    for (const auto& path : deduplicated_paths) {
         file_exists_futures.push_back(
             Via(executor_.get(), [this, path]() -> Result> {
                 PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(path));
@@ -235,12 +504,13 @@ Result>> FileStoreCommitImpl::F
     }
 
     for (size_t i = 1; i < committables.size(); i++) {
-        if (committables[i]->Identifier() < committables[i - 1]->Identifier()) {
+        if (committables[i]->Identifier() <= committables[i - 1]->Identifier()) {
             return Status::Invalid(
                 "Committables must be sorted according to identifiers before filtering. This is "
                 "unexpected.");
         }
     }
+    // TODO(yonghao.fyh): support commit strict mode last safe snapshot
     PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot,
                            snapshot_manager_->LatestSnapshotOfUser(commit_user_));
     if (latest_snapshot) {
@@ -250,7 +520,7 @@ Result>> FileStoreCommitImpl::F
             if (committable->Identifier() > latest_snapshot.value().CommitIdentifier()) {
                 result.push_back(committable);
             } else {
-                // TODO(yonghao.fyh): callback
+                // TODO(yonghao.fyh): support callback
             }
         }
         return result;
@@ -261,189 +531,340 @@ Result>> FileStoreCommitImpl::F
 }
 
 Status FileStoreCommitImpl::Overwrite(
-    const std::vector>& partitions,
+    const std::map& partition,
     const std::vector>& commit_messages, int64_t identifier,
     std::optional watermark) {
     std::shared_ptr committable =
         CreateManifestCommittable(identifier, commit_messages, watermark);
-    std::vector append_table_files;
-    std::vector append_changelog_files;
-    std::vector compact_table_files;
-    std::vector compact_changelog_files;
-    std::vector append_table_index_files;
-    std::vector compact_table_index_files;
-    PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files,
-                                        &append_changelog_files, &compact_table_files,
-                                        &compact_changelog_files, &append_table_index_files,
-                                        &compact_table_index_files));
-    if (!append_table_index_files.empty()) {
-        return Status::NotImplemented("Overwrite not support index for now");
-    }
-    return TryOverwrite(partitions, append_table_files, identifier, watermark);
+    PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu",
+                    table_name_.c_str(), committable->FileCommittables().size());
+    PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString());
+    std::string partition_str = fmt::format("{}", partition);
+    std::string properties_str = fmt::format("{}", committable->Properties());
+    PAIMON_LOG_DEBUG(logger_,
+                     "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: "
+                     "%s",
+                     partition_str.c_str(), committable_str.c_str(), properties_str.c_str());
+
+    Duration duration;
+    int32_t generated_snapshot = 0;
+    int32_t attempt = 0;
+
+    std::vector> partitions;
+    if (!partition.empty()) {
+        partitions.push_back(partition);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes,
+                           CollectChanges(committable->FileCommittables()));
+    ManifestEntryChanges report_changes = changes;
+    report_changes.append_changelog.clear();
+    report_changes.compact_changelog.clear();
+    ScopeGuard report_guard([&]() {
+        PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms",
+                        table_name_.c_str(), duration.Get());
+        ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt);
+    });
+
+    PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark, committable,
+                                          &generated_snapshot, &attempt));
+    return Status::OK();
 }
 
 Result FileStoreCommitImpl::FilterAndOverwrite(
-    const std::vector>& partitions,
+    const std::map& partition,
     const std::vector>& commit_messages, int64_t identifier,
     std::optional watermark) {
     std::shared_ptr committable =
         CreateManifestCommittable(identifier, commit_messages, watermark);
+    PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu",
+                    table_name_.c_str(), committable->FileCommittables().size());
+    PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString());
+    std::string partition_str = fmt::format("{}", partition);
+    std::string properties_str = fmt::format("{}", committable->Properties());
+    PAIMON_LOG_DEBUG(logger_,
+                     "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: "
+                     "%s",
+                     partition_str.c_str(), committable_str.c_str(), properties_str.c_str());
+
+    Duration duration;
+    int32_t generated_snapshot = 0;
+    int32_t attempt = 0;
+
+    std::vector> partitions;
+    if (!partition.empty()) {
+        partitions.push_back(partition);
+    }
+
     std::vector> committables;
     committables.push_back(committable);
     PAIMON_ASSIGN_OR_RAISE(std::vector> actual_committables,
                            FilterCommitted(committables));
     if (!actual_committables.empty()) {
-        std::vector append_table_files;
-        std::vector append_changelog_files;
-        std::vector compact_table_files;
-        std::vector compact_changelog_files;
-        std::vector append_table_index_files;
-        std::vector compact_table_index_files;
-        PAIMON_RETURN_NOT_OK(CollectChanges(actual_committables[0]->FileCommittables(),
-                                            &append_table_files, &append_changelog_files,
-                                            &compact_table_files, &compact_changelog_files,
-                                            &append_table_index_files, &compact_table_index_files));
-        if (!append_table_index_files.empty()) {
-            return Status::NotImplemented("FilterAndOverwrite not support index for now");
-        }
-        PAIMON_RETURN_NOT_OK(TryOverwrite(partitions, append_table_files, identifier, watermark));
+        PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes,
+                               CollectChanges(actual_committables[0]->FileCommittables()));
+        ManifestEntryChanges report_changes = changes;
+        report_changes.append_changelog.clear();
+        report_changes.compact_changelog.clear();
+        ScopeGuard report_guard([&]() {
+            PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms",
+                            table_name_.c_str(), duration.Get());
+            ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt);
+        });
+
+        PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark,
+                                              actual_committables[0], &generated_snapshot,
+                                              &attempt));
+    } else {
+        // Align with Java: filtered duplicate is treated as one resolved commit attempt.
+        attempt = 1;
+        PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms",
+                        table_name_.c_str(), duration.Get());
     }
     return actual_committables.size();
 }
 
+Status FileStoreCommitImpl::ExecuteOverwrite(
+    const std::vector>& partitions,
+    ManifestEntryChanges* changes, int64_t identifier, std::optional watermark,
+    const std::shared_ptr& committable, int32_t* generated_snapshot,
+    int32_t* attempt) {
+    if (!changes->append_changelog.empty() || !changes->compact_changelog.empty()) {
+        std::string warning =
+            "Overwrite mode currently does not commit any changelog.\n"
+            "Please make sure that the partition you're overwriting is not being consumed by a "
+            "streaming reader.\n"
+            "Ignored changelog files are:\n";
+        for (const auto& entry : changes->append_changelog) {
+            warning += fmt::format("  * {}\n", entry.ToString());
+        }
+        for (const auto& entry : changes->compact_changelog) {
+            warning += fmt::format("  * {}\n", entry.ToString());
+        }
+        PAIMON_LOG_WARN(logger_, "%s", warning.c_str());
+    }
+
+    bool skip_overwrite = false;
+    std::vector> overwrite_partitions = partitions;
+    if (!table_schema_->PartitionKeys().empty() && options_.DynamicPartitionOverwrite()) {
+        if (changes->append_table_files.empty()) {
+            // in dynamic mode, if there is no changes to commit, no data will be deleted
+            skip_overwrite = true;
+        } else {
+            std::set> dynamic_partitions;
+            for (const auto& entry : changes->append_table_files) {
+                std::map partition_map;
+                PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition()));
+                dynamic_partitions.insert(std::move(partition_map));
+            }
+            overwrite_partitions.assign(dynamic_partitions.begin(), dynamic_partitions.end());
+        }
+    } else if (!partitions.empty()) {
+        for (const auto& entry : changes->append_table_files) {
+            std::map partition_map;
+            PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition()));
+            bool belongs_to_overwrite_partition = false;
+            for (const auto& partition_spec : partitions) {
+                if (MatchPartitionSpec(partition_map, partition_spec)) {
+                    belongs_to_overwrite_partition = true;
+                    break;
+                }
+            }
+            if (!belongs_to_overwrite_partition) {
+                return Status::Invalid(fmt::format(
+                    "Trying to overwrite partitions {}, but the changes in {} does not belong to "
+                    "this partition",
+                    partitions, partition_map));
+            }
+        }
+    }
+
+    bool with_compact =
+        !changes->compact_table_files.empty() || !changes->compact_index_files.empty();
+    if (!with_compact) {
+        // In overwrite mode, opportunistically promote non-overlapping L0 files (per
+        // partition+bucket) to higher levels to reduce future compaction and read amplification,
+        // without changing semantics.
+        PAIMON_ASSIGN_OR_RAISE(changes->append_table_files,
+                               TryUpgrade(changes->append_table_files));
+    }
+
+    // overwrite new files
+    if (!skip_overwrite) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t cnt,
+                               TryOverwrite(overwrite_partitions, changes->append_table_files,
+                                            changes->append_index_files, identifier, watermark,
+                                            committable->Properties()));
+        *attempt += cnt;
+        *generated_snapshot += 1;
+    }
+
+    if (with_compact) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t cnt,
+                               TryCommit(changes->compact_table_files, /*changelog_files=*/{},
+                                         changes->compact_index_files, identifier, watermark,
+                                         committable->Properties(), Snapshot::CommitKind::Compact(),
+                                         /*detect_conflicts=*/true));
+        *attempt += cnt;
+        *generated_snapshot += 1;
+    }
+
+    return Status::OK();
+}
+
 Result FileStoreCommitImpl::GetLastCommitTableRequest() {
     return snapshot_commit_->GetLastCommitTableRequest();
 }
 
 Result> FileStoreCommitImpl::GetAllFiles(
-    const Snapshot& snapshot, const std::vector>& partitions) {
-    auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions,
-                                                    /*bucket_filter=*/std::nullopt);
-    PAIMON_ASSIGN_OR_RAISE(
-        auto scan, AppendOnlyFileStoreScan::Create(
-                       snapshot_manager_, schema_manager_, manifest_list_, manifest_file_,
-                       table_schema_, schema_, scan_filter, options_, executor_, memory_pool_));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan,
-                           scan->WithSnapshot(snapshot)->CreatePlan());
-    // scan existing file metas
-    return plan->Files();
+    const Snapshot& snapshot,
+    const std::vector>& partitions) const {
+    return commit_scanner_->ReadAllEntriesFromPartitions(snapshot, partitions);
 }
 
-Status FileStoreCommitImpl::TryOverwrite(
-    const std::vector>& partitions,
-    const std::vector& changes, int64_t commit_identifier,
-    std::optional watermark) {
-    int32_t retry_count = 0;
-    while (true) {
-        PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot,
-                               snapshot_manager_->LatestSnapshot());
-        std::vector changes_with_overwrite;
-        if (latest_snapshot) {
-            PAIMON_ASSIGN_OR_RAISE(std::vector entries,
-                                   GetAllFiles(latest_snapshot.value(), partitions));
-            for (const auto& entry : entries) {
-                changes_with_overwrite.emplace_back(FileKind::Delete(), entry.Partition(),
-                                                    entry.Bucket(), entry.TotalBuckets(),
-                                                    entry.File());
+Result> FileStoreCommitImpl::PartitionToMap(
+    const BinaryRow& partition) const {
+    std::vector> part_values;
+    PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer_->GeneratePartitionVector(partition));
+    std::map partition_map;
+    for (const auto& [key, value] : part_values) {
+        partition_map[key] = value;
+    }
+    return partition_map;
+}
+
+Result> FileStoreCommitImpl::TryUpgrade(
+    const std::vector& append_files) const {
+    if (!options_.OverwriteUpgrade()) {
+        return append_files;
+    }
+
+    if (table_schema_->PrimaryKeys().empty()) {
+        return append_files;
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys,
+                           table_schema_->TrimmedPrimaryKeys());
+    PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields,
+                           table_schema_->GetFields(trimmed_primary_keys));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator,
+                           FieldsComparator::Create(trimmed_primary_key_fields,
+                                                    options_.SequenceFieldSortOrderIsAscending()));
+
+    for (const auto& entry : append_files) {
+        if (entry.Level() > 0 || entry.Bucket() < 0) {
+            return append_files;
+        }
+    }
+
+    std::unordered_map, std::vector> buckets;
+    for (const auto& entry : append_files) {
+        buckets[std::make_pair(entry.Partition(), entry.Bucket())].emplace_back(entry);
+    }
+
+    std::vector results;
+    int32_t max_level = options_.GetNumLevels() - 1;
+    for (auto& [_, entries] : buckets) {
+        std::vector new_entries = entries;
+        std::sort(new_entries.begin(), new_entries.end(),
+                  [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) {
+                      return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0;
+                  });
+
+        bool overlap = false;
+        for (size_t i = 0; i + 1 < new_entries.size(); ++i) {
+            if (key_comparator->CompareTo(new_entries[i].MaxKey(), new_entries[i + 1].MinKey()) >=
+                0) {
+                overlap = true;
+                break;
             }
         }
-        changes_with_overwrite.insert(changes_with_overwrite.end(), changes.begin(), changes.end());
-        PAIMON_ASSIGN_OR_RAISE(bool commit_success,
-                               TryCommitOnce(changes_with_overwrite, /*index_entries=*/{},
-                                             commit_identifier, watermark,
-                                             /*log_offsets=*/{}, /*properties=*/{},
-                                             Snapshot::CommitKind::Overwrite(), latest_snapshot,
-                                             /*need_conflict_check=*/true));
-        if (commit_success) {
-            break;
+
+        if (overlap) {
+            results.insert(results.end(), entries.begin(), entries.end());
+            continue;
         }
-        if (retry_count >= options_.GetCommitMaxRetries()) {
-            return Status::Invalid(
-                fmt::format("Commit failed after {} attempts, there maybe exist commit conflicts "
-                            "between multiple jobs.",
-                            options_.GetCommitMaxRetries()));
+
+        PAIMON_LOG_INFO(logger_, "%s", "Upgraded for overwrite commit.");
+        for (const auto& entry : new_entries) {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr upgraded_file,
+                                   entry.File()->Upgrade(max_level));
+            results.emplace_back(entry.Kind(), entry.Partition(), entry.Bucket(),
+                                 entry.TotalBuckets(), upgraded_file);
         }
-        retry_count++;
     }
-    return Status::OK();
+
+    return results;
+}
+
+Result FileStoreCommitImpl::TryOverwrite(
+    const std::vector>& partitions,
+    const std::vector& changes, const std::vector& index_entries,
+    int64_t commit_identifier, std::optional watermark,
+    const std::map& properties) {
+    std::shared_ptr changes_provider =
+        commit_scanner_->OverwriteChangesProvider(partitions, changes, index_entries);
+    return TryCommit(changes_provider, commit_identifier, watermark, properties,
+                     Snapshot::CommitKind::Overwrite(), /*detect_conflicts=*/true);
 }
 
 Status FileStoreCommitImpl::Commit(const std::shared_ptr& committable,
                                    bool check_append_files) {
-    std::vector append_table_files;
-    std::vector append_changelog_files;
-    std::vector compact_table_files;
-    std::vector compact_changelog_files;
-    std::vector append_table_index_files;
-    std::vector compact_table_index_files;
-    PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files,
-                                        &append_changelog_files, &compact_table_files,
-                                        &compact_changelog_files, &append_table_index_files,
-                                        &compact_table_index_files));
+    PAIMON_LOG_INFO(logger_, "Ready to commit to table %s, number of commit messages: %zu",
+                    table_name_.c_str(), committable->FileCommittables().size());
+    PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString());
+    PAIMON_LOG_DEBUG(logger_, "Ready to commit\n%s", committable_str.c_str());
 
-    int32_t attempt = 0;
-    int32_t generated_snapshot = 0;
     Duration duration;
-    if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) {
-        PAIMON_ASSIGN_OR_RAISE(int32_t cnt,
-                               TryCommit(append_table_files, append_table_index_files,
-                                         committable->Identifier(), committable->Watermark(),
-                                         committable->LogOffsets(), committable->Properties(),
-                                         Snapshot::CommitKind::Append(), check_append_files));
+    int32_t generated_snapshot = 0;
+    int32_t attempt = 0;
+
+    PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes,
+                           CollectChanges(committable->FileCommittables()));
+    ScopeGuard report_guard([&]() {
+        PAIMON_LOG_INFO(logger_,
+                        "Finished (Uncertain of success) commit to table %s, duration %ld ms",
+                        table_name_.c_str(), duration.Get());
+        ReportCommit(changes, duration.Get(), generated_snapshot, attempt);
+    });
+
+    if (!ignore_empty_commit_ || changes.HasAppendChanges()) {
+        Snapshot::CommitKind commit_kind = Snapshot::CommitKind::Append();
+        if (append_commit_check_conflict_) {
+            check_append_files = true;
+        }
+
+        if (conflict_detection_.ShouldBeOverwriteCommit(changes.append_table_files,
+                                                        changes.append_index_files)) {
+            commit_kind = Snapshot::CommitKind::Overwrite();
+            check_append_files = true;
+        }
+        if (conflict_detection_.HasRowIdCheckFromSnapshot()) {
+            check_append_files = true;
+        }
+        if (changes.HasGlobalIndexFileAdditions()) {
+            check_append_files = true;
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(
+            int32_t cnt, TryCommit(changes.append_table_files, changes.append_changelog,
+                                   changes.append_index_files, committable->Identifier(),
+                                   committable->Watermark(), committable->Properties(), commit_kind,
+                                   check_append_files));
         attempt += cnt;
-        ++generated_snapshot;
+        generated_snapshot += 1;
     }
 
-    if (!compact_table_files.empty() || !compact_table_index_files.empty()) {
-        PAIMON_ASSIGN_OR_RAISE(
-            int32_t cnt, TryCommit(compact_table_files, compact_table_index_files,
-                                   committable->Identifier(), committable->Watermark(),
-                                   committable->LogOffsets(), committable->Properties(),
-                                   Snapshot::CommitKind::Compact(), /*check_append_files=*/true));
+    if (changes.HasCompactChanges()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t cnt,
+                               TryCommit(changes.compact_table_files, changes.compact_changelog,
+                                         changes.compact_index_files, committable->Identifier(),
+                                         committable->Watermark(), committable->Properties(),
+                                         Snapshot::CommitKind::Compact(),
+                                         /*detect_conflicts=*/true));
         attempt += cnt;
-        ++generated_snapshot;
-    }
-    auto table_files_added = static_cast(append_table_files.size());
-    int32_t table_files_deleted = 0;
-    int64_t compaction_input_file_size = 0;
-    int64_t compaction_output_file_size = 0;
-    for (const auto& entry : compact_table_files) {
-        const auto& kind = entry.Kind();
-        if (kind == FileKind::Add()) {
-            ++table_files_added;
-            compaction_output_file_size += entry.File()->file_size;
-        } else if (kind == FileKind::Delete()) {
-            ++table_files_deleted;
-            compaction_input_file_size += entry.File()->file_size;
-        }
-    }
-    metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get());
-    metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt);
-    metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added);
-    metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted);
-    metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED, append_table_files.size());
-    metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED,
-                         compact_table_files.size());
-    metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED,
-                         append_changelog_files.size());
-    metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED,
-                         compact_changelog_files.size());
-    metrics_->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS, generated_snapshot);
-    metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED, RowCounts(append_table_files));
-    metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED,
-                         RowCounts(append_changelog_files));
-    metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED,
-                         RowCounts(compact_table_files));
-    metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED,
-                         RowCounts(compact_changelog_files));
-    metrics_->SetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN,
-                         NumChangedPartitions({append_table_files, compact_table_files}));
-    metrics_->SetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN,
-                         NumChangedBuckets({append_table_files, compact_table_files}));
-    metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE,
-                         compaction_input_file_size);
-    metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE,
-                         compaction_output_file_size);
+        generated_snapshot += 1;
+    }
     return Status::OK();
 }
 
@@ -456,21 +877,34 @@ Status FileStoreCommitImpl::Commit(
 }
 
 Result FileStoreCommitImpl::TryCommit(const std::vector& delta_files,
+                                               const std::vector& changelog_files,
                                                const std::vector& index_entries,
                                                int64_t identifier, std::optional watermark,
-                                               std::map log_offsets,
                                                const std::map& properties,
                                                Snapshot::CommitKind commit_kind,
-                                               bool check_append_files) {
+                                               bool detect_conflicts) {
+    std::shared_ptr changes_provider =
+        CommitChangesProvider::Provider(delta_files, changelog_files, index_entries);
+    return TryCommit(changes_provider, identifier, watermark, properties, commit_kind,
+                     detect_conflicts);
+}
+
+Result FileStoreCommitImpl::TryCommit(
+    const std::shared_ptr& changes_provider, int64_t identifier,
+    std::optional watermark, const std::map& properties,
+    Snapshot::CommitKind commit_kind, bool detect_conflicts) {
     int32_t retry_count = 0;
     int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000;
     while (true) {
         PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot,
                                snapshot_manager_->LatestSnapshot());
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr commit_changes,
+                               changes_provider->Provide(latest_snapshot));
         PAIMON_ASSIGN_OR_RAISE(
             bool commit_success,
-            TryCommitOnce(delta_files, index_entries, identifier, watermark, log_offsets,
-                          properties, commit_kind, latest_snapshot, check_append_files));
+            TryCommitOnce(commit_changes->delta_files, commit_changes->changelog_files,
+                          commit_changes->index_entries, identifier, watermark, properties,
+                          commit_kind, latest_snapshot, detect_conflicts));
         if (commit_success) {
             break;
         }
@@ -480,90 +914,90 @@ Result FileStoreCommitImpl::TryCommit(const std::vector&
             return Status::Invalid(
                 fmt::format("Commit failed after {} millis with {} retries, there maybe exist "
                             "commit conflicts between multiple jobs.",
-                            options_.GetCommitTimeout(), options_.GetCommitMaxRetries()));
+                            options_.GetCommitTimeout(), retry_count));
         }
+        retry_waiter_.RetryWait(retry_count);
         retry_count++;
     }
     return retry_count + 1;
 }
 
-Result>> FileStoreCommitImpl::ChangedPartitions(
-    const std::vector& data_files,
-    const std::vector& index_files) const {
-    std::set> partitions;
-    auto add_partition = [&, this](const BinaryRow& partition_row) -> Status {
-        std::vector> part_values;
-        PAIMON_ASSIGN_OR_RAISE(part_values,
-                               partition_computer_->GeneratePartitionVector(partition_row));
-        if (part_values.empty()) {
-            return Status::OK();
-        }
-        std::map part_values_map;
-        for (const auto& [key, value] : part_values) {
-            part_values_map[key] = value;
-        }
-        partitions.insert(part_values_map);
+Status FileStoreCommitImpl::CheckSameBucketFromSnapshot(
+    const std::vector& delta_entries,
+    const std::optional& latest_snapshot) const {
+    if (!latest_snapshot) {
         return Status::OK();
-    };
+    }
 
-    for (const ManifestEntry& entry : data_files) {
-        PAIMON_RETURN_NOT_OK(add_partition(entry.Partition()));
+    std::unordered_map expected_total_buckets;
+    PAIMON_RETURN_NOT_OK(conflict_detection_.CollectUncheckedBucketPartitions(
+        delta_entries, &expected_total_buckets));
+    if (expected_total_buckets.empty()) {
+        return Status::OK();
     }
-    for (const IndexManifestEntry& entry : index_files) {
-        if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) {
-            PAIMON_RETURN_NOT_OK(add_partition(entry.partition));
-        }
+
+    std::vector changed_partitions;
+    changed_partitions.reserve(expected_total_buckets.size());
+    for (const auto& [partition, _] : expected_total_buckets) {
+        changed_partitions.push_back(partition);
     }
-    return partitions;
-}
 
-Result> FileStoreCommitImpl::ReadAllEntriesFromChangedPartitions(
-    const Snapshot& latest_snapshot,
-    const std::set>& partitions) const {
-    std::vector> partition_filters(partitions.begin(),
-                                                                      partitions.end());
-    auto scan_filter = std::make_shared(/*predicate=*/nullptr, partition_filters,
-                                                    /*bucket_filter=*/std::nullopt);
     PAIMON_ASSIGN_OR_RAISE(
-        auto scan, AppendOnlyFileStoreScan::Create(
-                       snapshot_manager_, schema_manager_, manifest_list_, manifest_file_,
-                       table_schema_, schema_, scan_filter, options_, executor_, memory_pool_));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan,
-                           scan->WithSnapshot(latest_snapshot)->CreatePlan());
-    // scan existing file metas
-    return plan->Files();
-}
-
-Status FileStoreCommitImpl::NoConflictsOrFail(const std::string& base_commit_user,
-                                              const std::vector& base_entries,
-                                              const std::vector& changes) const {
-    ScopeGuard guard([&]() {
-        PAIMON_LOG_WARN(logger_, "File deletion conflicts detected! Give up committing. %s",
-                        base_commit_user.c_str());
-    });
-    std::vector all_entries = base_entries;
-    all_entries.insert(all_entries.end(), changes.begin(), changes.end());
-    std::vector merged_entries;
-    PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries));
-    for (const auto& entry : merged_entries) {
-        if (entry.Kind() == FileKind::Delete()) {
-            return Status::Invalid(fmt::format(
-                "Trying to delete file {} which is not previously added.", entry.FileName()));
+        auto previous_total_buckets,
+        commit_scanner_->ReadTotalBuckets(latest_snapshot.value(), changed_partitions));
+
+    return conflict_detection_.CheckSameBucketByTotalBuckets(expected_total_buckets,
+                                                             previous_total_buckets);
+}
+
+bool FileStoreCommitImpl::ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const {
+    return commit_kind == Snapshot::CommitKind::Append() &&
+           bucket_mode_ == BucketMode::HASH_FIXED &&
+           (IsUnorderedWriteOnlyAppend() || IsWriteOnlySnapshotSequenceAppend());
+}
+
+bool FileStoreCommitImpl::IsUnorderedWriteOnlyAppend() const {
+    return options_.WriteOnly() && !options_.BucketAppendOrdered();
+}
+
+bool FileStoreCommitImpl::IsWriteOnlySnapshotSequenceAppend() const {
+    return options_.WriteOnly() &&
+           options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT;
+}
+
+Result> FileStoreCommitImpl::MaxSequenceNumber(
+    const std::vector& manifests) const {
+    int64_t max_from_manifest = std::numeric_limits::min();
+    bool found = false;
+    for (const auto& manifest : manifests) {
+        std::vector entries;
+        PAIMON_RETURN_NOT_OK(manifest_file_->Read(
+            manifest.FileName(), [](const ManifestEntry&) -> Result { return true; },
+            &entries));
+        std::optional current_max =
+            SequenceSnapshotProperties::MaxSequenceNumberFromFiles(entries);
+        if (current_max) {
+            max_from_manifest = std::max(max_from_manifest, current_max.value());
+            found = true;
         }
     }
-    // TODO(yonghao.fyh): check for all LSM level >= 1, key ranges of files do not intersect
-    guard.Release();
-    return Status::OK();
+
+    if (!found) {
+        return std::optional();
+    }
+    return std::optional(max_from_manifest);
 }
 
 Result FileStoreCommitImpl::TryCommitOnce(
     const std::vector& delta_entries,
+    const std::vector& changelog_entries,
     const std::vector& index_entries, int64_t identifier,
-    std::optional watermark, std::map log_offsets,
-    const std::map& properties, Snapshot::CommitKind commit_kind,
-    const std::optional& latest_snapshot, bool need_conflict_check) {
+    std::optional watermark, const std::map& properties,
+    Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot,
+    bool detect_conflicts) {
     std::vector delta_files = delta_entries;
     int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000;
+
     int64_t new_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID;
     int64_t first_row_id_start = 0;
     if (latest_snapshot) {
@@ -574,19 +1008,68 @@ Result FileStoreCommitImpl::TryCommitOnce(
         }
     }
 
-    PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot #%ld", new_snapshot_id);
+    // TODO(yonghao.fyh): support strict mode checker
+
+    PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot %ld", new_snapshot_id);
     for (const ManifestEntry& entry : delta_files) {
         PAIMON_LOG_DEBUG(logger_, "  * %s", entry.ToString().c_str());
     }
+    PAIMON_LOG_DEBUG(logger_, "Ready to commit changelog files to snapshot %ld", new_snapshot_id);
+    for (const ManifestEntry& entry : changelog_entries) {
+        PAIMON_LOG_DEBUG(logger_, "  * %s", entry.ToString().c_str());
+    }
 
-    if (need_conflict_check && latest_snapshot) {
-        std::set> changed_partitions;
-        PAIMON_ASSIGN_OR_RAISE(changed_partitions, ChangedPartitions(delta_files, index_entries));
-        PAIMON_ASSIGN_OR_RAISE(
-            std::vector base_data_files,
-            ReadAllEntriesFromChangedPartitions(latest_snapshot.value(), changed_partitions));
-        PAIMON_RETURN_NOT_OK(
-            NoConflictsOrFail(latest_snapshot.value().CommitUser(), base_data_files, delta_files));
+    bool discard_duplicate =
+        options_.CommitDiscardDuplicateFiles() && commit_kind == Snapshot::CommitKind::Append();
+    bool check_conflicts = latest_snapshot.has_value() && (discard_duplicate || detect_conflicts);
+    // By default, if checkConflicts is required, we do not have to do the extra check bucket
+    // here.
+    if (!check_conflicts && ShouldCheckSameBucket(commit_kind)) {
+        PAIMON_RETURN_NOT_OK(CheckSameBucketFromSnapshot(delta_files, latest_snapshot));
+    }
+
+    if (check_conflicts) {
+        // latest snapshot id is different from the snapshot id we've checked for conflicts,
+        // so we have to check again
+        std::vector changed_partitions =
+            ManifestEntryChanges::ChangedPartitions(delta_files, index_entries);
+        PAIMON_ASSIGN_OR_RAISE(std::vector base_data_files,
+                               commit_scanner_->ReadAllEntriesFromChangedPartitions(
+                                   latest_snapshot.value(), changed_partitions));
+
+        if (discard_duplicate) {
+            std::unordered_set base_identifiers;
+            base_identifiers.reserve(base_data_files.size());
+            for (const auto& entry : base_data_files) {
+                base_identifiers.insert(entry.CreateIdentifier());
+            }
+
+            delta_files.erase(
+                std::remove_if(delta_files.begin(), delta_files.end(),
+                               [&base_identifiers](const ManifestEntry& entry) {
+                                   return base_identifiers.find(entry.CreateIdentifier()) !=
+                                          base_identifiers.end();
+                               }),
+                delta_files.end());
+        }
+
+        std::optional> row_id_column_conflict_checker =
+            std::nullopt;
+        if (conflict_detection_.HasRowIdCheckFromSnapshot()) {
+            std::vector> delta_data_files;
+            delta_data_files.reserve(delta_files.size());
+            for (const auto& entry : delta_files) {
+                delta_data_files.push_back(entry.File());
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr checker,
+                RowIdColumnConflictChecker::FromDataFiles(schema_manager_, delta_data_files));
+            row_id_column_conflict_checker = checker;
+        }
+
+        PAIMON_RETURN_NOT_OK(conflict_detection_.CheckConflicts(
+            latest_snapshot.value(), base_data_files, delta_files, index_entries,
+            row_id_column_conflict_checker, commit_kind));
     }
 
     std::vector merge_before_manifests;
@@ -616,25 +1099,13 @@ Result FileStoreCommitImpl::TryCommitOnce(
 
     if (latest_snapshot) {
         old_index_manifest = latest_snapshot.value().IndexManifest();
-        // TODO(yonghao.fyh): total record count should call scan when its std::nullopt
-        previous_total_record_count = latest_snapshot.value().TotalRecordCount() != std::nullopt
-                                          ? latest_snapshot.value().TotalRecordCount().value()
-                                          : 0;
+        previous_total_record_count = latest_snapshot.value().TotalRecordCount();
         std::vector previous_manifests;
         // read all previous manifest files
         PAIMON_RETURN_NOT_OK(
             manifest_list_->ReadDataManifests(latest_snapshot.value(), &previous_manifests));
         merge_before_manifests.insert(merge_before_manifests.end(), previous_manifests.begin(),
                                       previous_manifests.end());
-        // read the last snapshot to complete the bucket's offsets when logOffsets does not
-        // contain all buckets
-        std::optional> latest_log_offsets =
-            latest_snapshot.value().LogOffsets();
-        if (latest_log_offsets) {
-            for (const auto& [key, value] : latest_log_offsets.value()) {
-                log_offsets.emplace(key, value);
-            }
-        }
         std::optional latest_watermark = latest_snapshot.value().Watermark();
         if (latest_watermark) {
             if (watermark == std::nullopt) {
@@ -668,11 +1139,12 @@ Result FileStoreCommitImpl::TryCommitOnce(
                                    std::make_move_iterator(entries.end()));
             }
         }
-        // assigned snapshot id to delta files
-        AssignSnapshotId(new_snapshot_id, &delta_files);
-        // assign row id for new files
-        PAIMON_ASSIGN_OR_RAISE(next_row_id_start,
-                               AssignRowTrackingMeta(first_row_id_start, &delta_files));
+
+        PAIMON_ASSIGN_OR_RAISE(RowTrackingCommitUtils::RowTrackingAssigned assigned,
+                               RowTrackingCommitUtils::AssignRowTracking(
+                                   new_snapshot_id, first_row_id_start, delta_files));
+        next_row_id_start = assigned.next_row_id_start;
+        delta_files = std::move(assigned.assigned_entries);
     }
 
     // the added records subtract the deleted records from
@@ -693,12 +1165,21 @@ Result FileStoreCommitImpl::TryCommitOnce(
                                  new_changes_manifests.end());
     PAIMON_ASSIGN_OR_RAISE(delta_manifest_list, manifest_list_->Write(new_changes_manifests));
 
+    // write changelog into manifest files
+    std::optional> changelog_manifest_list;
+    if (!changelog_entries.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::vector changelog_manifests,
+                               manifest_file_->Write(changelog_entries));
+        PAIMON_ASSIGN_OR_RAISE(changelog_manifest_list, manifest_list_->Write(changelog_manifests));
+    }
+
     PAIMON_ASSIGN_OR_RAISE(index_manifest_name, index_manifest_file_->WriteIndexFiles(
                                                     old_index_manifest, index_entries));
 
-    std::optional> changelog_manifest_list;
-    std::optional statistics;
-    int64_t changelog_record_count = 0;
+    std::optional statistics =
+        latest_snapshot ? latest_snapshot.value().Statistics() : std::nullopt;
+    std::optional changelog_record_count =
+        ManifestEntry::NullableRecordCount(changelog_entries);
     int64_t schema_id = 0;
     PAIMON_ASSIGN_OR_RAISE(std::optional> table_schema,
                            schema_manager_->Latest());
@@ -706,6 +1187,25 @@ Result FileStoreCommitImpl::TryCommitOnce(
         schema_id = table_schema.value()->Id();
     }
 
+    // Keep Java semantics: inherit previous stats only when schema matches.
+    if (statistics && latest_snapshot && latest_snapshot.value().SchemaId() != schema_id) {
+        PAIMON_LOG_WARN(logger_, "%s", "Schema changed, stats will not be inherited");
+        statistics = std::nullopt;
+    }
+
+    std::map snapshot_properties = properties;
+    if (options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT) {
+        PAIMON_ASSIGN_OR_RAISE(std::optional latest_max_sequence_number,
+                               SequenceSnapshotProperties::MaxSequenceNumber(latest_snapshot));
+        if (!latest_max_sequence_number && latest_snapshot) {
+            PAIMON_ASSIGN_OR_RAISE(latest_max_sequence_number,
+                                   MaxSequenceNumber(merge_before_manifests));
+        }
+        snapshot_properties = SequenceSnapshotProperties::MergeMaxSequenceNumber(
+            snapshot_properties, latest_max_sequence_number, delta_files);
+    }
+
+    // prepare snapshot file
     Snapshot new_snapshot(
         new_snapshot_id, schema_id, base_manifest_list.first, base_manifest_list.second,
         delta_manifest_list.first, delta_manifest_list.second,
@@ -714,16 +1214,18 @@ Result FileStoreCommitImpl::TryCommitOnce(
         changelog_manifest_list ? std::optional(changelog_manifest_list.value().second)
                                 : std::nullopt,
         index_manifest_name, commit_user_, identifier, commit_kind,
-        DateTimeUtils::GetCurrentUTCTimeUs() / 1000, log_offsets, total_record_count,
-        delta_record_count, changelog_record_count, watermark, statistics,
-        properties.empty() ? std::nullopt
-                           : std::optional>(properties),
+        DateTimeUtils::GetCurrentUTCTimeUs() / 1000, total_record_count, delta_record_count,
+        changelog_record_count, watermark, statistics,
+        snapshot_properties.empty()
+            ? std::nullopt
+            : std::optional>(snapshot_properties),
         next_row_id_start);
 
     Result commit_result = CommitSnapshotImpl(new_snapshot, delta_statistics);
     if (!commit_result.ok()) {
         // commit exception, not sure about the situation and should not clean up the files.
-        PAIMON_LOG_WARN(logger_, "You need call FilterAndCommit to retry commit for exception. %s",
+        PAIMON_LOG_WARN(logger_,
+                        "You need to call FilterAndCommit to retry commit for exception. %s",
                         commit_result.status().ToString().c_str());
 
         // To prevent the case where an atomic write times out but actually succeeds,
@@ -734,7 +1236,7 @@ Result FileStoreCommitImpl::TryCommitOnce(
         // as it may delete meta files from a snapshot that was just written by ourselves,
         // leading to an incomplete or corrupted snapshot.
         guard.Release();
-        return Status::Invalid("You need call FilterAndCommit to retry commit for exception. ",
+        return Status::Invalid("You need to call FilterAndCommit to retry commit for exception. ",
                                commit_result.status().ToString());
     }
     bool commit_success = commit_result.value();
@@ -745,6 +1247,7 @@ Result FileStoreCommitImpl::TryCommitOnce(
                         new_snapshot.Id(), root_path_.c_str(), commit_user_.c_str(),
                         new_snapshot.CommitIdentifier(),
                         Snapshot::CommitKind::ToString(new_snapshot.GetCommitKind()).c_str());
+        last_committed_snapshot_id_ = new_snapshot.Id();
         guard.Release();
         return true;
     } else {
@@ -753,70 +1256,6 @@ Result FileStoreCommitImpl::TryCommitOnce(
     }
 }
 
-void FileStoreCommitImpl::AssignSnapshotId(int64_t snapshot_id,
-                                           std::vector* delta_files) const {
-    for (auto& entry : *delta_files) {
-        entry.AssignSequenceNumber(/*min_sequence_number=*/snapshot_id,
-                                   /*max_sequence_number=*/snapshot_id);
-    }
-}
-
-Result FileStoreCommitImpl::AssignRowTrackingMeta(
-    int64_t first_row_id_start, std::vector* delta_files) const {
-    if (delta_files->empty()) {
-        return first_row_id_start;
-    }
-    // assign row id for new files
-    int64_t start = first_row_id_start;
-    int64_t blob_start_default = first_row_id_start;
-    // Per-blob-field row id tracking: each blob field maintains its own start position,
-    // keyed by the blob field name (from write_cols[0]).
-    std::map blob_starts;
-    // TODO(xinyu.lxy): support vector store file row tracking when vector store is implemented
-    for (auto& entry : *delta_files) {
-        if (entry.File()->file_source == std::nullopt) {
-            return Status::Invalid(
-                "This is a bug, file source field for row-tracking table must present.");
-        }
-        bool contains_row_id =
-            entry.File()->write_cols.has_value() &&
-            std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(),
-                      SpecialFields::RowId().Name()) != entry.File()->write_cols->end();
-        if (entry.File()->file_source.value() == FileSource::Append() &&
-            entry.File()->first_row_id == std::nullopt && !contains_row_id) {
-            int64_t row_count = entry.File()->row_count;
-            if (BlobUtils::IsBlobFile(entry.File()->file_name)) {
-                // Use the first write_col as the blob field name to support
-                // independent row tracking per blob field.
-                std::string blob_field_name;
-                if (!entry.File()->write_cols || entry.File()->write_cols->empty()) {
-                    return Status::Invalid(fmt::format(
-                        "invalid blob file {}: does not have write_cols", entry.File()->file_name));
-                }
-                blob_field_name = entry.File()->write_cols->at(0);
-                int64_t blob_start = blob_starts.count(blob_field_name)
-                                         ? blob_starts[blob_field_name]
-                                         : blob_start_default;
-                if (blob_start >= start) {
-                    return Status::Invalid(fmt::format(
-                        "This is a bug, blob start {} should be less than start {} when "
-                        "assigning a blob entry file.",
-                        blob_start, start));
-                }
-                entry.AssignFirstRowId(blob_start);
-                blob_starts[blob_field_name] = blob_start + row_count;
-            } else {
-                entry.AssignFirstRowId(start);
-                blob_start_default = start;
-                blob_starts.clear();
-                start += row_count;
-            }
-        }
-        // for compact file, do not assign first row id.
-    }
-    return start;
-}
-
 Result FileStoreCommitImpl::CommitSnapshotImpl(
     const Snapshot& new_snapshot, const std::vector& delta_statistics) {
     std::vector statistics;
@@ -885,118 +1324,24 @@ std::shared_ptr FileStoreCommitImpl::CreateManifestCommitta
     return committable;
 }
 
-Status FileStoreCommitImpl::CollectChanges(
-    const std::vector>& commit_messages,
-    std::vector* append_table_files,
-    std::vector* append_changelog_files,
-    std::vector* compact_table_files,
-    std::vector* compact_changelog_files,
-    std::vector* append_table_index_files,
-    std::vector* compact_table_index_files) {
+Result FileStoreCommitImpl::CollectChanges(
+    const std::vector>& commit_messages) {
+    ManifestEntryChanges changes(num_bucket_);
     for (const auto& message : commit_messages) {
-        auto commit_message = std::dynamic_pointer_cast(message);
-        if (commit_message) {
-            DataIncrement new_files_increment = commit_message->GetNewFilesIncrement();
-            for (const std::shared_ptr& new_file : new_files_increment.NewFiles()) {
-                append_table_files->push_back(MakeEntry(FileKind::Add(), commit_message, new_file));
-            }
-            for (const std::shared_ptr& deleted_file :
-                 new_files_increment.DeletedFiles()) {
-                append_table_files->push_back(
-                    MakeEntry(FileKind::Delete(), commit_message, deleted_file));
-            }
-            for (const std::shared_ptr& changelog_file :
-                 new_files_increment.ChangelogFiles()) {
-                append_changelog_files->push_back(
-                    MakeEntry(FileKind::Add(), commit_message, changelog_file));
-            }
-            for (const std::shared_ptr& deleted_index_file :
-                 new_files_increment.DeletedIndexFiles()) {
-                append_table_index_files->emplace_back(
-                    FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(),
-                    deleted_index_file);
-            }
-            for (const std::shared_ptr& new_index_file :
-                 new_files_increment.NewIndexFiles()) {
-                append_table_index_files->emplace_back(FileKind::Add(), commit_message->Partition(),
-                                                       commit_message->Bucket(), new_index_file);
-            }
-            CompactIncrement compact_increment = commit_message->GetCompactIncrement();
-            for (const std::shared_ptr& compact_before :
-                 compact_increment.CompactBefore()) {
-                compact_table_files->push_back(
-                    MakeEntry(FileKind::Delete(), commit_message, compact_before));
-            }
-            for (const std::shared_ptr& compact_after :
-                 compact_increment.CompactAfter()) {
-                compact_table_files->push_back(
-                    MakeEntry(FileKind::Add(), commit_message, compact_after));
-            }
-            for (const std::shared_ptr& changelog_file :
-                 compact_increment.ChangelogFiles()) {
-                compact_changelog_files->push_back(
-                    MakeEntry(FileKind::Add(), commit_message, changelog_file));
-            }
-            for (const std::shared_ptr& deleted_index_file :
-                 compact_increment.DeletedIndexFiles()) {
-                compact_table_index_files->emplace_back(
-                    FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(),
-                    deleted_index_file);
-            }
-            for (const std::shared_ptr& new_index_file :
-                 compact_increment.NewIndexFiles()) {
-                compact_table_index_files->emplace_back(FileKind::Add(),
-                                                        commit_message->Partition(),
-                                                        commit_message->Bucket(), new_index_file);
-            }
-        } else {
-            return Status::Invalid("fail to cast commit message to commit message impl");
-        }
+        PAIMON_RETURN_NOT_OK(changes.Collect(message));
     }
-    return Status::OK();
-}
-
-ManifestEntry FileStoreCommitImpl::MakeEntry(
-    const FileKind& kind, const std::shared_ptr& commit_message,
-    const std::shared_ptr& file) const {
-    int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt
-                                ? num_bucket_
-                                : commit_message->TotalBuckets().value();
-    return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets,
-                         file);
+    PAIMON_LOG_INFO(logger_, "Finished collecting changes, including: %s",
+                    changes.ToString().c_str());
+    return changes;
 }
 
-int64_t FileStoreCommitImpl::RowCounts(const std::vector& files) {
-    return std::accumulate(files.begin(), files.end(), 0L,
-                           [](int64_t row_count, const ManifestEntry& entry) {
-                               return row_count + entry.File()->row_count;
-                           });
-}
-
-int64_t FileStoreCommitImpl::NumChangedPartitions(
-    const std::vector>& changes) {
-    std::unordered_set changed_partitions;
-    for (const auto& change : changes) {
-        for (const auto& entry : change) {
-            changed_partitions.insert(entry.Partition());
-        }
-    }
-    return static_cast(changed_partitions.size());
-}
-
-int64_t FileStoreCommitImpl::NumChangedBuckets(
-    const std::vector>& changes) {
-    std::unordered_map> changed_partition_buckets;
-    for (const auto& change : changes) {
-        for (const auto& entry : change) {
-            changed_partition_buckets[entry.Partition()].insert(entry.Bucket());
-        }
-    }
-    return std::accumulate(changed_partition_buckets.begin(), changed_partition_buckets.end(),
-                           int64_t{0}, [](int64_t num_changed_buckets, const auto& bucket) {
-                               return num_changed_buckets +
-                                      static_cast(bucket.second.size());
-                           });
+void FileStoreCommitImpl::ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration,
+                                       int32_t generated_snapshot, int32_t attempt) {
+    CommitStats commit_stats(changes.append_table_files, changes.append_changelog,
+                             changes.compact_table_files, changes.compact_changelog,
+                             commit_duration, generated_snapshot, attempt,
+                             last_committed_snapshot_id_);
+    CommitMetrics::ReportCommit(metrics_, commit_stats);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h
index 217edc06..c0c93c9a 100644
--- a/src/paimon/core/operation/file_store_commit_impl.h
+++ b/src/paimon/core/operation/file_store_commit_impl.h
@@ -31,7 +31,13 @@
 #include "paimon/core/catalog/snapshot_commit.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/manifest/partition_entry.h"
+#include "paimon/core/operation/commit/commit_scanner.h"
+#include "paimon/core/operation/commit/conflict_detection.h"
+#include "paimon/core/operation/commit/manifest_entry_changes.h"
+#include "paimon/core/operation/commit/retry_waiter.h"
+#include "paimon/core/operation/commit/row_id_column_conflict_checker.h"
 #include "paimon/core/snapshot.h"
+#include "paimon/core/table/bucket_mode.h"
 #include "paimon/file_store_commit.h"
 #include "paimon/logging.h"
 #include "paimon/memory/memory_pool.h"
@@ -63,6 +69,7 @@ class SnapshotManager;
 class SchemaManager;
 class TableSchema;
 class BinaryRowPartitionComputer;
+class CommitChangesProvider;
 class CommitMessage;
 class Executor;
 class FileSystem;
@@ -75,6 +82,8 @@ class SnapshotCommit;
 /// Commit operation which provides commit and overwrite.
 class FileStoreCommitImpl : public FileStoreCommit {
  public:
+    static Status ValidateCommitOptions(const CoreOptions& options);
+
     FileStoreCommitImpl(const std::shared_ptr& pool,
                         const std::shared_ptr& executor,
                         const std::shared_ptr& schema, const std::string& root_path,
@@ -83,12 +92,14 @@ class FileStoreCommitImpl : public FileStoreCommit {
                         std::unique_ptr partition_computer,
                         const std::shared_ptr& snapshot_manager,
                         bool ignore_empty_commit, bool use_rest_catalog_commit,
+                        bool append_commit_check_conflict,
                         const std::shared_ptr& table_schema,
                         const std::shared_ptr& manifest_file,
                         const std::shared_ptr& manifest_list,
                         const std::shared_ptr& index_manifest_file,
                         const std::shared_ptr& expire_snapshots,
-                        const std::shared_ptr& schema_manager);
+                        const std::shared_ptr& schema_manager,
+                        CommitScanner::ScanSupplier scan_supplier);
     ~FileStoreCommitImpl() override;
 
     Status Commit(const std::vector>& commit_messages,
@@ -100,13 +111,13 @@ class FileStoreCommitImpl : public FileStoreCommit {
             commit_identifier_and_messages,
         std::optional watermark = std::nullopt) override;
 
-    Status Overwrite(const std::vector>& partitions,
+    Status Overwrite(const std::map& partition,
                      const std::vector>& commit_messages,
                      int64_t commit_identifier,
                      std::optional watermark = std::nullopt) override;
 
     Result FilterAndOverwrite(
-        const std::vector>& partitions,
+        const std::map& partition,
         const std::vector>& commit_messages,
         int64_t commit_identifier, std::optional watermark = std::nullopt) override;
 
@@ -117,6 +128,14 @@ class FileStoreCommitImpl : public FileStoreCommit {
     Status DropPartition(const std::vector>& partitions,
                          int64_t commit_identifier) override;
 
+    Status TruncateTable(int64_t commit_identifier) override;
+
+    Status Abort(const std::vector>& commit_messages) override;
+
+    Result RollbackToAsLatest(int64_t target_snapshot_id) override;
+
+    FileStoreCommit& RowIdCheckConflict(std::optional row_id_check_from_snapshot) override;
+
     std::shared_ptr GetCommitMetrics() const override {
         return metrics_;
     }
@@ -127,13 +146,26 @@ class FileStoreCommitImpl : public FileStoreCommit {
     Status Commit(const std::shared_ptr& manifest_committable,
                   bool check_append_files);
 
-    Status TryOverwrite(const std::vector>& partition,
-                        const std::vector& changes, int64_t commit_identifier,
-                        std::optional watermark);
+    Result TryOverwrite(const std::vector>& partition,
+                                 const std::vector& changes,
+                                 const std::vector& index_entries,
+                                 int64_t commit_identifier, std::optional watermark,
+                                 const std::map& properties);
+
+    Status ExecuteOverwrite(const std::vector>& partitions,
+                            ManifestEntryChanges* changes, int64_t identifier,
+                            std::optional watermark,
+                            const std::shared_ptr& committable,
+                            int32_t* generated_snapshot, int32_t* attempt);
 
     Result> GetAllFiles(
         const Snapshot& snapshot,
-        const std::vector>& partitions);
+        const std::vector>& partitions) const;
+
+    Result> PartitionToMap(const BinaryRow& partition) const;
+
+    Result> TryUpgrade(
+        const std::vector& append_files) const;
 
     Result>> FilterCommitted(
         const std::vector>& committables);
@@ -142,36 +174,38 @@ class FileStoreCommitImpl : public FileStoreCommit {
         int64_t identifier, const std::vector>& commit_messages,
         std::optional watermark);
 
-    ManifestEntry MakeEntry(const FileKind& kind,
-                            const std::shared_ptr& commit_message,
-                            const std::shared_ptr& file) const;
+    Result CollectChanges(
+        const std::vector>& commit_messages);
 
-    Status CollectChanges(const std::vector>& commit_messages,
-                          std::vector* append_table_files,
-                          std::vector* append_changelog_files,
-                          std::vector* compact_table_files,
-                          std::vector* compact_changelog_files,
-                          std::vector* append_table_index_files,
-                          std::vector* compact_table_index_files);
+    void ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration,
+                      int32_t generated_snapshot, int32_t attempt);
 
     Result TryCommit(const std::vector& delta_files,
+                              const std::vector& changelog_files,
                               const std::vector& index_entries,
                               int64_t identifier, std::optional watermark,
-                              std::map log_offsets,
                               const std::map& properties,
-                              Snapshot::CommitKind commit_kind, bool check_append_files);
+                              Snapshot::CommitKind commit_kind, bool detect_conflicts);
+
+    Result TryCommit(const std::shared_ptr& changes_provider,
+                              int64_t identifier, std::optional watermark,
+                              const std::map& properties,
+                              Snapshot::CommitKind commit_kind, bool detect_conflicts);
+
     Result TryCommitOnce(const std::vector& delta_files,
+                               const std::vector& changelog_files,
                                const std::vector& index_entries,
                                int64_t commit_identifier, std::optional watermark,
-                               std::map log_offsets,
                                const std::map& properties,
                                Snapshot::CommitKind commit_kind,
                                const std::optional& latest_snapshot,
-                               bool need_conflict_check);
+                               bool detect_conflicts);
 
     Result CommitSnapshotImpl(const Snapshot& new_snapshot,
                                     const std::vector& delta_statistics);
 
+    Result> ReadAddManifestEntries(const Snapshot& snapshot) const;
+
     void CleanUpTmpManifests(const std::string& previous_changes_list_name,
                              const std::string& new_changes_list_name,
                              const std::vector& old_metas,
@@ -179,37 +213,27 @@ class FileStoreCommitImpl : public FileStoreCommit {
                              const std::optional& old_index_manifest,
                              const std::optional& new_index_manifest);
 
-    Result> ReadAllEntriesFromChangedPartitions(
-        const Snapshot& latest_snapshot,
-        const std::set>& partitions) const;
-
-    Status NoConflictsOrFail(const std::string& base_commit_user,
-                             const std::vector& base_entries,
-                             const std::vector& changes) const;
-
-    Status CheckFilesExistence(
-        const std::vector>& committables) const;
-
-    void AssignSnapshotId(int64_t snapshot_id, std::vector* delta_files) const;
+    Status CheckSameBucketFromSnapshot(const std::vector& delta_entries,
+                                       const std::optional& latest_snapshot) const;
 
-    Result AssignRowTrackingMeta(int64_t first_row_id_start,
-                                          std::vector* delta_files) const;
+    bool ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const;
 
-    Result>> ChangedPartitions(
-        const std::vector& data_files,
-        const std::vector& index_files) const;
+    bool IsUnorderedWriteOnlyAppend() const;
 
-    static int64_t RowCounts(const std::vector& files);
+    bool IsWriteOnlySnapshotSequenceAppend() const;
 
-    static int64_t NumChangedPartitions(const std::vector>& changes);
+    Result> MaxSequenceNumber(
+        const std::vector& manifests) const;
 
-    static int64_t NumChangedBuckets(const std::vector>& changes);
+    Status CheckFilesExistence(
+        const std::vector>& committables) const;
 
  private:
     std::shared_ptr memory_pool_;
     std::shared_ptr executor_;
     std::shared_ptr schema_;
     std::string root_path_;
+    std::string table_name_;
     std::string commit_user_;
     CoreOptions options_;
     std::shared_ptr path_factory_;
@@ -219,8 +243,13 @@ class FileStoreCommitImpl : public FileStoreCommit {
     std::shared_ptr snapshot_manager_;
     std::shared_ptr snapshot_commit_;
     bool ignore_empty_commit_ = true;
+    bool append_commit_check_conflict_ = false;
+    RetryWaiter retry_waiter_;
     int32_t num_bucket_ = 0;
+    BucketMode bucket_mode_ = BucketMode::BUCKET_UNAWARE;
     std::shared_ptr table_schema_;
+    std::shared_ptr commit_scanner_;
+    ConflictDetection conflict_detection_;
 
     std::shared_ptr manifest_file_;
     std::shared_ptr manifest_list_;
@@ -231,6 +260,7 @@ class FileStoreCommitImpl : public FileStoreCommit {
 
     std::shared_ptr metrics_;
     std::shared_ptr logger_;
+    int64_t last_committed_snapshot_id_ = -1;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp
index be42387b..405b8280 100644
--- a/src/paimon/core/operation/file_store_commit_impl_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp
@@ -39,20 +39,34 @@
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
 #include "paimon/common/factories/io_hook.h"
+#include "paimon/common/utils/linked_hash_map.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/catalog/commit_table_request.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/index/global_index_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/index/index_path_factory.h"
+#include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/io/data_file_path_factory.h"
+#include "paimon/core/io/data_increment.h"
 #include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/file_source.h"
 #include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/index_manifest_file.h"
 #include "paimon/core/manifest/manifest_committable.h"
 #include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/operation/metrics/commit_metrics.h"
 #include "paimon/core/partition/partition_statistics.h"
+#include "paimon/core/schema/table_schema.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/file_utils.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/data/timestamp.h"
@@ -61,6 +75,7 @@
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/fs/local/local_file_system_factory.h"
+#include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/metrics.h"
 #include "paimon/testing/utils/binary_row_generator.h"
@@ -69,6 +84,7 @@
 #include "paimon/testing/utils/timezone_guard.h"
 
 namespace paimon::test {
+
 class GmockFileSystem : public LocalFileSystem {
  public:
     MOCK_METHOD(Status, ReadFile, (const std::string& path, std::string* content), (override));
@@ -78,6 +94,7 @@ class GmockFileSystem : public LocalFileSystem {
                 (const, override));
     MOCK_METHOD(Status, AtomicStore, (const std::string& path, const std::string& content),
                 (override));
+    MOCK_METHOD(Result, Exists, (const std::string& path), (const, override));
 };
 
 class GmockFileSystemFactory : public LocalFileSystemFactory {
@@ -111,6 +128,11 @@ class GmockFileSystemFactory : public LocalFileSystemFactory {
                 return fs_ptr->FileSystem::AtomicStore(path, content);
             }));
 
+        ON_CALL(*fs, Exists(A()))
+            .WillByDefault(Invoke([fs_ptr](const std::string& path) {
+                return fs_ptr->LocalFileSystem::Exists(path);
+            }));
+
         return fs;
     }
 };
@@ -160,11 +182,18 @@ class FileStoreCommitImplTest : public testing::Test {
 
     ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition,
                                       const FileKind& kind) const {
+        return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(),
+                                   DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0);
+    }
+
+    ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition,
+                                      const FileKind& kind, const BinaryRow& min_key,
+                                      const BinaryRow& max_key, int32_t level, int32_t bucket = 0,
+                                      int32_t total_buckets = 2) const {
         auto data_file_meta = std::make_shared(
-            file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
-            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
-            /*max_seq_no=*/32,
-            /*schema_id=*/1, /*level=*/2,
+            file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(),
+            SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32,
+            /*schema_id=*/1, level,
             /*extra_files=*/std::vector>(),
             /*creation_time=*/Timestamp(0, 0),
             /*delete_row_count=*/3,
@@ -172,7 +201,15 @@ class FileStoreCommitImplTest : public testing::Test {
             /*external_path=*/std::nullopt,
             /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt,
             /*write_cols=*/std::nullopt);
-        return ManifestEntry(kind, partition, 0, 2, data_file_meta);
+        return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta);
+    }
+
+    BinaryRow CreateIntRow(int32_t value) const {
+        BinaryRow row(1);
+        BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+        writer.WriteInt(0, value);
+        writer.Complete();
+        return row;
     }
 
     ManifestEntry CreateManifestEntryWithNoPartition(const std::string& file_name,
@@ -200,6 +237,68 @@ class FileStoreCommitImplTest : public testing::Test {
         return result;
     }
 
+    size_t CountFiles(const std::string& dir) const {
+        size_t count = 0;
+        std::error_code ec;
+        for (std::filesystem::directory_iterator it(dir, ec), end; it != end; it.increment(ec)) {
+            if (it->is_regular_file(ec)) {
+                ++count;
+            }
+        }
+        return count;
+    }
+
+    std::shared_ptr CreateIndexFileMeta(const std::string& file_name,
+                                                       const std::string& index_type = "bitmap") {
+        return std::make_shared(index_type, file_name, /*file_size=*/100,
+                                               /*row_count=*/5, /*dv_ranges=*/std::nullopt,
+                                               /*external_path=*/std::nullopt,
+                                               /*global_index_meta=*/std::nullopt);
+    }
+
+    std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name,
+                                                             int64_t row_range_start,
+                                                             int64_t row_range_end) {
+        GlobalIndexMeta global_index(row_range_start, row_range_end, /*index_field_id=*/1,
+                                     /*extra_field_ids=*/std::nullopt,
+                                     std::make_shared("meta", GetDefaultPool().get()));
+        return std::make_shared(
+            "HASH", file_name, /*file_size=*/100, /*row_count=*/5,
+            /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index);
+    }
+
+    std::shared_ptr CreateLeveledDataFileMeta(const std::string& file_name,
+                                                            const BinaryRow& min_key,
+                                                            const BinaryRow& max_key, int32_t level,
+                                                            int32_t schema_id = 0) {
+        return std::make_shared(
+            file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(),
+            SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, schema_id, level,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt);
+    }
+
+    std::shared_ptr CreateAppendDataFileMeta(const std::string& file_name,
+                                                           int64_t row_count) {
+        return std::make_shared(
+            file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+            SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16,
+            /*max_seq_no=*/32,
+            /*schema_id=*/1, /*level=*/2,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr, FileSource::Append(),
+            /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt);
+    }
+
     bool IsStringInSet(const std::set& strSet, const std::string& target) {
         return strSet.find(target) != strSet.end();
     }
@@ -255,8 +354,9 @@ class FileStoreCommitImplTest : public testing::Test {
 
         EXPECT_OK_AND_ASSIGN(std::unique_ptr in_stream, file_system->Open(path));
         EXPECT_TRUE(in_stream);
-        EXPECT_OK_AND_ASSIGN([[maybe_unused]] int32_t length,
-                             in_stream->Read(reinterpret_cast(buffer.data()), buffer.size()))
+        EXPECT_OK_AND_ASSIGN(
+            [[maybe_unused]] int64_t length,
+            in_stream->Read(reinterpret_cast(buffer.data()), buffer.size()));
         EXPECT_OK(in_stream->Close());
         auto pool = GetDefaultPool();
 
@@ -339,9 +439,9 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
         /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt,
         /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807,
         /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597,
-        /*log_offsets=*/std::map(), /*total_record_count=*/5,
-        /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt,
-        /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0);
+        /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/std::nullopt,
+        /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt,
+        /*next_row_id=*/0);
     std::vector expected_partition_statistics = {
         PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541,
                             /*file_count=*/1,
@@ -365,6 +465,48 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) {
     ASSERT_FALSE(exist);
 }
 
+TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs1 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    ASSERT_GT(msgs1.size(), 0);
+    ASSERT_OK(commit_impl->Commit(msgs1, 1));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_TRUE(snapshot1.Properties());
+    auto iter1 = snapshot1.Properties().value().find("sequence.generation.max-sequence-number");
+    ASSERT_TRUE(iter1 != snapshot1.Properties().value().end());
+    int64_t max_seq_1 = std::stoll(iter1->second);
+
+    std::vector> msgs2 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-02",
+                          /*version=*/3);
+    ASSERT_GT(msgs2.size(), 0);
+    ASSERT_OK(commit_impl->Commit(msgs2, 2));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot2, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_TRUE(snapshot2.Properties());
+    auto iter2 = snapshot2.Properties().value().find("sequence.generation.max-sequence-number");
+    ASSERT_TRUE(iter2 != snapshot2.Properties().value().end());
+    int64_t max_seq_2 = std::stoll(iter2->second);
+
+    ASSERT_GE(max_seq_2, max_seq_1);
+}
+
 TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) {
     std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/";
     auto dir = UniqueTestDirectory::Create();
@@ -377,6 +519,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes)
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
                              .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
                              .AddOption(Options::COMMIT_MAX_RETRIES, "10")
+                             .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms")
+                             .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms")
                              .WithFileSystem(fs)
                              .Finish());
 
@@ -417,25 +561,25 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) {
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
                              .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms")
+                             .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms")
                              .WithFileSystem(fs)
                              .Finish());
 
     ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
     std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST");
     auto* mock_fs = dynamic_cast(fs.get());
+    EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_))
+        .Times(testing::AnyNumber())
+        .WillRepeatedly(testing::Invoke([&](const std::string& path, std::string* content) {
+            return mock_fs->FileSystem::ReadFile(path, content);
+        }));
     EXPECT_CALL(*mock_fs, ReadFile(testing::StrEq(latest_hint), testing::_))
         .WillRepeatedly(testing::Invoke([](const std::string& path, std::string* content) {
             *content = "-1";
             return Status::OK();
         }));
 
-    EXPECT_CALL(
-        *mock_fs,
-        ReadFile(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot/snapshot-5")), testing::_))
-        .WillOnce(testing::Invoke([&](const std::string& path, std::string* content) {
-            return mock_fs->FileSystem::ReadFile(path, content);
-        }));
-
     EXPECT_CALL(*mock_fs, ListDir(testing::_, testing::_)).Times(testing::AnyNumber());
     EXPECT_CALL(*mock_fs,
                 ListDir(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot")), testing::_))
@@ -582,7 +726,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) {
         ASSERT_TRUE(metrics);
         ASSERT_OK_AND_ASSIGN(uint64_t counter,
                              metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
-        ASSERT_EQ(1u, counter);
+        ASSERT_EQ(0u, counter);
         ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(PathUtil::JoinPath(
                                              table_path_, "snapshot/snapshot-3")));
         ASSERT_FALSE(exist);
@@ -661,6 +805,224 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) {
     }
 }
 
+TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+
+    // Append three times so the latest snapshot (3) is a superset of snapshot 1.
+    const std::string base_path = paimon::test::GetDataDir() +
+                                  "/orc/append_09.db/append_09/commit_messages/commit_messages-0";
+    for (int32_t i = 1; i <= 3; ++i) {
+        std::vector> msgs =
+            GetCommitMessages(base_path + std::to_string(i), /*version=*/3);
+        ASSERT_GT(msgs.size(), 0);
+        ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/i));
+    }
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot3, commit_impl->snapshot_manager_->LoadSnapshot(3));
+    ASSERT_OK_AND_ASSIGN(std::vector snapshot1_entries,
+                         commit_impl->ReadAddManifestEntries(snapshot1));
+    ASSERT_OK_AND_ASSIGN(std::vector snapshot3_entries,
+                         commit_impl->ReadAddManifestEntries(snapshot3));
+
+    // Roll back to snapshot 1: the delta only removes files (DELETE branch).
+    ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1));
+    ASSERT_TRUE(rolled_back);
+    ASSERT_OK_AND_ASSIGN(bool snapshot4_exist, file_system_->Exists(PathUtil::JoinPath(
+                                                   table_path_, "snapshot/snapshot-4")));
+    ASSERT_TRUE(snapshot4_exist);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot4, commit_impl->snapshot_manager_->LoadSnapshot(4));
+    ASSERT_EQ(snapshot4.Id(), 4);
+    ASSERT_TRUE(snapshot4.GetCommitKind() == Snapshot::CommitKind::Overwrite());
+    ASSERT_EQ(snapshot4.SchemaId(), snapshot1.SchemaId());
+    ASSERT_EQ(snapshot4.TotalRecordCount(), snapshot1.TotalRecordCount());
+    ASSERT_EQ(snapshot4.NextRowId(), std::max(snapshot3.NextRowId(), snapshot1.NextRowId()));
+    ASSERT_EQ(snapshot4.IndexManifest(), snapshot1.IndexManifest());
+    ASSERT_OK_AND_ASSIGN(std::vector snapshot4_entries,
+                         commit_impl->ReadAddManifestEntries(snapshot4));
+    ASSERT_EQ(CollectFileNames(snapshot4_entries), CollectFileNames(snapshot1_entries));
+
+    // Roll back forward to snapshot 3: the delta only adds files (ADD branch).
+    ASSERT_OK_AND_ASSIGN(bool rolled_forward, commit->RollbackToAsLatest(/*target_snapshot_id=*/3));
+    ASSERT_TRUE(rolled_forward);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot5, commit_impl->snapshot_manager_->LoadSnapshot(5));
+    ASSERT_EQ(snapshot5.Id(), 5);
+    ASSERT_TRUE(snapshot5.GetCommitKind() == Snapshot::CommitKind::Overwrite());
+    ASSERT_EQ(snapshot5.TotalRecordCount(), snapshot3.TotalRecordCount());
+    ASSERT_EQ(snapshot5.IndexManifest(), snapshot3.IndexManifest());
+    ASSERT_OK_AND_ASSIGN(std::vector snapshot5_entries,
+                         commit_impl->ReadAddManifestEntries(snapshot5));
+    ASSERT_EQ(CollectFileNames(snapshot5_entries), CollectFileNames(snapshot3_entries));
+}
+
+TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1),
+                        "Latest snapshot is null, can not roll back.");
+}
+
+TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    ASSERT_GT(msgs.size(), 0);
+    ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1));
+    ASSERT_FALSE(commit->RollbackToAsLatest(/*target_snapshot_id=*/999).ok());
+}
+
+TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) {
+    // Mirror Java testRollbackToAsLatestDeletionVectorChangeIsInvisibleToStreaming: when the target
+    // and the latest snapshot share identical data files and differ only in the index (deletion
+    // vector) manifest, the rollback produces an empty data delta and the new snapshot inherits the
+    // target's index manifest.
+    std::map table_options = {{Options::FILE_FORMAT, "orc"},
+                                                        {Options::FILE_SYSTEM, "local"},
+                                                        {Options::BUCKET, "2"},
+                                                        {Options::BUCKET_KEY, "f2"}};
+    ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(test_root_, table_options));
+    arrow::Schema typed_schema(fields_);
+    ::ArrowSchema schema;
+    ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok());
+    ASSERT_OK(catalog->CreateTable(Identifier("foo", "dv_bar"), &schema,
+                                   /*partition_keys=*/{"f1"},
+                                   /*primary_keys=*/{}, table_options,
+                                   /*ignore_if_exists=*/false));
+    std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar");
+
+    CommitContextBuilder context_builder(dv_table_path, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+
+    BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get());
+
+    // snapshot 1 (target): a single data file, no deletion vectors.
+    std::vector> new_files;
+    new_files.push_back(CreateAppendDataFileMeta("data-file-1", /*row_count=*/10));
+    DataIncrement data_increment(std::move(new_files), {}, {});
+    std::shared_ptr data_msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit->Commit({data_msg}, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(Snapshot target_snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1));
+
+    // snapshot 2 (latest): a deletion-vector-only change on the same data file. The data file is
+    // unchanged, only the index manifest gains a deletion vector.
+    LinkedHashMap dv_ranges;
+    dv_ranges.insert_or_assign("data-file-1",
+                               DeletionVectorMeta(/*data_file_name=*/"data-file-1", /*offset=*/0,
+                                                  /*length=*/10, /*cardinality=*/1));
+    std::vector> new_index_files;
+    new_index_files.push_back(std::make_shared(
+        DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-index-1", /*file_size=*/100,
+        /*row_count=*/1, /*dv_ranges=*/dv_ranges, /*external_path=*/std::nullopt));
+    DataIncrement dv_increment({}, {}, {}, std::move(new_index_files), {});
+    std::shared_ptr dv_msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, dv_increment, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit->Commit({dv_msg}, /*commit_identifier=*/2));
+    ASSERT_OK_AND_ASSIGN(Snapshot latest_snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_TRUE(latest_snapshot.IndexManifest().has_value());
+
+    // Roll back to snapshot 1: the data files are identical, so the delta carries no data change
+    // and the new snapshot inherits the target's (empty) index manifest, dropping the deletion
+    // vector.
+    ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1));
+    ASSERT_TRUE(rolled_back);
+    ASSERT_OK_AND_ASSIGN(Snapshot rolled_back_snapshot,
+                         commit_impl->snapshot_manager_->LoadSnapshot(3));
+    ASSERT_EQ(rolled_back_snapshot.Id(), 3);
+    ASSERT_TRUE(rolled_back_snapshot.GetCommitKind() == Snapshot::CommitKind::Overwrite());
+    // Empty data delta: no records are added or removed by the rollback.
+    ASSERT_EQ(rolled_back_snapshot.DeltaRecordCount(), 0);
+    // The new snapshot points back to the target's index manifest.
+    ASSERT_EQ(rolled_back_snapshot.IndexManifest(), target_snapshot.IndexManifest());
+    // The surviving data files are unchanged relative to the target.
+    ASSERT_OK_AND_ASSIGN(std::vector target_entries,
+                         commit_impl->ReadAddManifestEntries(target_snapshot));
+    ASSERT_OK_AND_ASSIGN(std::vector rolled_back_entries,
+                         commit_impl->ReadAddManifestEntries(rolled_back_snapshot));
+    ASSERT_EQ(CollectFileNames(rolled_back_entries), CollectFileNames(target_entries));
+}
+
+TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsFalse) {
+    // When a concurrent writer wins the race for the next snapshot id, the atomic snapshot commit
+    // reports failure (returns false); RollbackToAsLatest surfaces that without error and without
+    // advancing the committed snapshot id. The temporary base/delta manifest lists written before
+    // the failed commit are intentionally left behind, matching Java (no cleanup on this path).
+    //
+    // The conflict is a TOCTOU race: snapshot-2 does not exist when LatestSnapshot() resolves the
+    // current latest (so it stays snapshot-1), but exists by the time RenamingSnapshotCommit checks
+    // before writing. A stateful Exists() mock reproduces exactly that ordering.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr fs,
+                         FileSystemFactory::Get("gmock_fs", table_path_, {}));
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .WithFileSystem(fs)
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    ASSERT_GT(msgs.size(), 0);
+    ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1));
+    ASSERT_EQ(commit_impl->last_committed_snapshot_id_, 1);
+
+    // snapshot-2 is invisible the first time it is probed (LatestSnapshot keeps snapshot-1 as the
+    // latest) and visible afterwards (RenamingSnapshotCommit sees the conflicting file).
+    const std::string next_snapshot = PathUtil::JoinPath(table_path_, "snapshot/snapshot-2");
+    auto* mock_fs = dynamic_cast(fs.get());
+    ASSERT_TRUE(mock_fs);
+    EXPECT_CALL(*mock_fs, Exists(testing::_))
+        .Times(testing::AnyNumber())
+        .WillRepeatedly(testing::Invoke(
+            [mock_fs](const std::string& path) { return mock_fs->LocalFileSystem::Exists(path); }));
+    EXPECT_CALL(*mock_fs, Exists(testing::StrEq(next_snapshot)))
+        .WillOnce(testing::Return(Result(false)))
+        .WillRepeatedly(testing::Return(Result(true)));
+
+    const std::string manifest_dir = PathUtil::JoinPath(table_path_, "manifest");
+    const size_t manifest_count_before = CountFiles(manifest_dir);
+
+    ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1));
+    ASSERT_FALSE(rolled_back);
+    // The committed snapshot id must not advance on the failed rollback.
+    ASSERT_EQ(commit_impl->last_committed_snapshot_id_, 1);
+    // The temporary base/delta manifest lists are leaked (not cleaned up) on the failure path.
+    ASSERT_GT(CountFiles(manifest_dir), manifest_count_before);
+}
+
 TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -739,18 +1101,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) {
         io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);
         auto status = commit->Commit(msgs);
         io_hook->Clear();
+        ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath(
+                                              table_path_, "snapshot/snapshot-2")));
         // termination conditions:
-        // 1. status does not hit IOHook, already touch all IO operation
-        // 2. hit IOHook in commit hint, at this point, the snapshot is already committed
-        if (!HitIOHook(status) || HitIOHookInCommitHint(status)) {
+        // 1. hit IOHook in commit hint, at this point, the snapshot is already committed
+        // 2. snapshot file exists, which means atomic store succeeded even if a later IO failed
+        if (HitIOHookInCommitHint(status) || exist2) {
             scanned_all_io_hook = true;
-            ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath(
-                                                  table_path_, "snapshot/snapshot-2")));
             ASSERT_TRUE(exist2);
             break;
         }
-        ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath(
-                                              table_path_, "snapshot/snapshot-2")));
+        // For some IO-hook positions, retries may be exhausted and return a generic non-IOHook
+        // status while the snapshot is still not committed. Keep scanning next positions.
+        if (!HitIOHook(status)) {
+            continue;
+        }
         ASSERT_FALSE(exist2);
         std::vector actual_snapshots;
         ASSERT_OK(
@@ -869,7 +1234,6 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) {
     ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(PathUtil::JoinPath(
                                     table_path_, "manifest/" + index_manifest.value())));
     ASSERT_FALSE(exist);
-
     commit_impl->CleanUpTmpManifests(
         snapshot.value().BaseManifestList(), snapshot.value().DeltaManifestList(), /*old_metas=*/{},
         /*new_metas=*/previous_manifests, /*old_index_manifest=*/std::nullopt,
@@ -900,133 +1264,6 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) {
     ASSERT_EQ(0u, counter);
 }
 
-TEST_F(FileStoreCommitImplTest, TestCheckConflict) {
-    CommitContextBuilder context_builder(table_path_, "commit_user_1");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
-                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
-                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
-                             .AddOption(Options::FILE_SYSTEM, "local")
-                             .IgnoreEmptyCommit(true)
-                             .Finish());
-
-    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
-    auto commit_impl = dynamic_cast(commit.get());
-    ASSERT_TRUE(commit_impl);
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Add()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file2", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Delete()));
-        ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file6", FileKind::Add()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file2", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Delete()));
-        ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Add()));
-        ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Add()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Add()));
-        ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Add()));
-        ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-
-        std::vector changes;
-        ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete()));
-
-        std::vector changes;
-        ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-    {
-        std::vector base_entries;
-        base_entries.push_back(CreateManifestEntry("file1", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file2", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file3", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file4", FileKind::Add()));
-        base_entries.push_back(CreateManifestEntry("file5", FileKind::Add()));
-
-        std::vector changes;
-        changes.push_back(CreateManifestEntry("file1", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file2", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file3", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file4", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file5", FileKind::Delete()));
-        changes.push_back(CreateManifestEntry("file6", FileKind::Delete()));
-        ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes));
-    }
-}
-
 TEST_F(FileStoreCommitImplTest, TestTryOverwrite) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -1053,8 +1290,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) {
     std::vector changes;
     changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add()));
     std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}};
-    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes,
-                                        /*commit_identifier=*/1, std::nullopt));
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{},
+                                        /*commit_identifier=*/1, std::nullopt,
+                                        /*properties=*/{}));
 }
 
 TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
@@ -1072,8 +1310,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
     std::vector changes;
     changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add()));
     std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}};
-    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes,
-                                        /*commit_identifier=*/0, std::nullopt));
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{},
+                                        /*commit_identifier=*/0, std::nullopt,
+                                        /*properties=*/{}));
     ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot());
     ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {}));
     ASSERT_EQ(1u, entries1.size());
@@ -1081,8 +1320,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
     ASSERT_EQ(FileKind::Add(), entries1[0].Kind());
     std::vector changes2;
     changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add()));
-    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2,
-                                        /*commit_identifier=*/1, std::nullopt));
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{},
+                                        /*commit_identifier=*/1, std::nullopt,
+                                        /*properties=*/{}));
     ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot());
     ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {}));
     ASSERT_EQ(1u, entries2.size());
@@ -1090,6 +1330,34 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) {
     ASSERT_EQ(FileKind::Add(), entries2[0].Kind());
 }
 
+TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .IgnoreEmptyCommit(true)
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+
+    std::vector changes;
+    changes.push_back(CreateManifestEntry("new_file_with_properties", FileKind::Add()));
+    std::vector> partitions = {{{"f1", "10"}}};
+    std::map properties = {{"overwrite-prop", "v1"}};
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{},
+                                        /*commit_identifier=*/0, std::nullopt, properties));
+
+    ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_TRUE(snapshot.has_value());
+    ASSERT_TRUE(snapshot->Properties().has_value());
+    auto iter = snapshot->Properties()->find("overwrite-prop");
+    ASSERT_TRUE(iter != snapshot->Properties()->end());
+    ASSERT_EQ("v1", iter->second);
+}
+
 TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -1105,8 +1373,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) {
     std::vector changes;
     changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add()));
     std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}};
-    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes,
-                                        /*commit_identifier=*/0, std::nullopt));
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{},
+                                        /*commit_identifier=*/0, std::nullopt,
+                                        /*properties=*/{}));
     std::vector> msgs =
         GetCommitMessages(paimon::test::GetDataDir() +
                               "/orc/append_09.db/append_09/commit_messages/"
@@ -1131,8 +1400,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) {
 
     std::vector changes2;
     changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add()));
-    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2,
-                                        /*commit_identifier=*/2, std::nullopt));
+    ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{},
+                                        /*commit_identifier=*/2, std::nullopt,
+                                        /*properties=*/{}));
     ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot());
     ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {}));
     ASSERT_EQ(1u, entries2.size());
@@ -1236,7 +1506,7 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) {
     ASSERT_EQ(3, manifests[0].NumDeletedFiles());
 }
 
-TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) {
+TEST_F(FileStoreCommitImplTest, TestTruncateTable) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
@@ -1245,113 +1515,284 @@ TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) {
                              .IgnoreEmptyCommit(true)
                              .Finish());
 
-    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
-    auto commit_impl = std::dynamic_pointer_cast(
-        std::shared_ptr(std::move(commit)));
-    std::vector> msgs;
-    auto committable = commit_impl->CreateManifestCommittable(1, msgs, 30);
-    ASSERT_TRUE(committable);
-    EXPECT_EQ(1, committable->Identifier());
-    EXPECT_EQ(30, committable->Watermark().value());
-    ASSERT_TRUE(IsEqualMsgs(msgs, committable->FileCommittables()));
-}
-
-TEST_F(FileStoreCommitImplTest, TestCollectChanges) {
-    CommitContextBuilder context_builder(table_path_, "commit_user_1");
-    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
-                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
-                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
-                             .AddOption(Options::FILE_SYSTEM, "local")
-                             .AddOption(Options::BUCKET, "10")
-                             .Finish());
-
     ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
     std::vector> msgs =
         GetCommitMessages(paimon::test::GetDataDir() +
                               "/orc/append_09.db/append_09/commit_messages/"
                               "commit_messages-01",
                           /*version=*/3);
-    auto commit_impl = std::dynamic_pointer_cast(
-        std::shared_ptr(std::move(commit)));
-    std::vector append_table_files;
-    std::vector append_changelog_files;
-    std::vector compact_table_files;
-    std::vector compact_changelog_files;
-    std::vector append_table_index_files;
-    std::vector compact_table_index_files;
-    ASSERT_OK(commit_impl->CollectChanges(msgs, &append_table_files, &append_changelog_files,
-                                          &compact_table_files, &compact_changelog_files,
-                                          &append_table_index_files, &compact_table_index_files));
-    ASSERT_EQ(append_table_files.size(), 3u);
-    ASSERT_EQ(append_changelog_files.size(), 0u);
-    ASSERT_EQ(compact_table_files.size(), 0u);
-    ASSERT_EQ(compact_changelog_files.size(), 0u);
-    ASSERT_EQ(append_table_index_files.size(), 0u);
-    ASSERT_EQ(compact_table_index_files.size(), 0u);
-    ASSERT_EQ(append_table_files[0].Kind(), FileKind::Add());
-    ASSERT_EQ(append_table_files[0].Bucket(), 0);
-    ASSERT_EQ(append_table_files[0].TotalBuckets(), 10);
-    ASSERT_EQ(append_table_files[0].Level(), 0);
-    ASSERT_EQ(append_table_files[0].FileName(), "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc");
-    ASSERT_EQ(append_table_files[1].Kind(), FileKind::Add());
-    ASSERT_EQ(append_table_files[1].Bucket(), 1);
-    ASSERT_EQ(append_table_files[1].TotalBuckets(), 10);
-    ASSERT_EQ(append_table_files[1].Level(), 0);
-    ASSERT_EQ(append_table_files[1].FileName(), "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc");
-    ASSERT_EQ(append_table_files[2].Kind(), FileKind::Add());
-    ASSERT_EQ(append_table_files[2].Bucket(), 0);
-    ASSERT_EQ(append_table_files[2].TotalBuckets(), 10);
-    ASSERT_EQ(append_table_files[2].Level(), 0);
-    ASSERT_EQ(append_table_files[2].FileName(), "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc");
+    ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/0));
+
+    // Truncate overwrites all partitions with no new files, deleting every existing file.
+    ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(table_path_ + "/snapshot/snapshot-2"));
+    ASSERT_TRUE(exist);
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.GetCommitKind());
+    std::vector manifests;
+    ASSERT_OK(commit_impl->manifest_list_->ReadDeltaManifests(snapshot, &manifests));
+    ASSERT_EQ(1, manifests.size());
+    ASSERT_EQ(0, manifests[0].NumAddedFiles());
+    // The append_09 fixture contains 3 data files spread across partitions f1=10 and f1=20.
+    ASSERT_EQ(3, manifests[0].NumDeletedFiles());
 }
 
-TEST_F(FileStoreCommitImplTest, TestFilterCommitted) {
+TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
-                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
                              .AddOption(Options::FILE_SYSTEM, "local")
                              .Finish());
-
     ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
     auto commit_impl = std::dynamic_pointer_cast(
         std::shared_ptr(std::move(commit)));
 
-    std::vector> msgs =
-        GetCommitMessages(paimon::test::GetDataDir() +
-                              "/orc/append_09.db/append_09/commit_messages/"
-                              "commit_messages-01",
-                          /*version=*/3);
-    auto committable = commit_impl->CreateManifestCommittable(1, msgs, std::nullopt);
-    std::vector> committables = {committable};
+    const BinaryRow partition = CreateIntRow(10);
+    const int32_t bucket = 0;
+    auto new_data_file = CreateAppendDataFileMeta("abort-new-data", 1);
+    auto compact_data_file = CreateAppendDataFileMeta("abort-compact-data", 1);
+    auto new_index_file = CreateIndexFileMeta("abort-new-index");
+    auto compact_index_file = CreateIndexFileMeta("abort-compact-index");
+
+    DataIncrement data_increment(/*new_files=*/{new_data_file}, /*deleted_files=*/{},
+                                 /*changelog_files=*/{}, /*new_index_files=*/{new_index_file},
+                                 /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{compact_data_file},
+                                       /*changelog_files=*/{},
+                                       /*new_index_files=*/{compact_index_file},
+                                       /*deleted_index_files=*/{});
+    std::shared_ptr message = std::make_shared(
+        partition, bucket, /*total_buckets=*/2, data_increment, compact_increment);
+
+    // Materialize the referenced files so we can observe them being cleaned up.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr data_pf,
+                         commit_impl->path_factory_->CreateDataFilePathFactory(partition, bucket));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr index_pf,
+                         commit_impl->path_factory_->CreateIndexFileFactory(partition, bucket));
+    std::vector paths = {
+        data_pf->ToPath(new_data_file), data_pf->ToPath(compact_data_file),
+        index_pf->ToPath(new_index_file), index_pf->ToPath(compact_index_file)};
+    for (const auto& path : paths) {
+        ASSERT_OK(file_system_->WriteFile(path, /*content=*/"", /*overwrite=*/false));
+        ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(path));
+        ASSERT_TRUE(exist);
+    }
 
-    // Test with no previous snapshots
-    ASSERT_OK_AND_ASSIGN(auto filtered_committables, commit_impl->FilterCommitted(committables));
-    ASSERT_EQ(filtered_committables.size(), committables.size());
+    ASSERT_OK(commit_impl->Abort({message}));
 
-    // Test with a previous snapshot
-    ASSERT_OK(commit_impl->Commit(committable, /*check_append_files=*/false));
-    ASSERT_OK_AND_ASSIGN(filtered_committables, commit_impl->FilterCommitted(committables));
-    ASSERT_EQ(filtered_committables.size(), 0);
+    for (const auto& path : paths) {
+        ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(path));
+        ASSERT_FALSE(exist);
+    }
 }
 
-TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) {
+TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
-                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
                              .AddOption(Options::FILE_SYSTEM, "local")
                              .Finish());
-
     ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
     auto commit_impl = std::dynamic_pointer_cast(
         std::shared_ptr(std::move(commit)));
 
-    std::vector> msgs1 =
-        GetCommitMessages(paimon::test::GetDataDir() +
-                              "/orc/append_09.db/append_09/commit_messages/"
-                              "commit_messages-01",
-                          /*version=*/3);
+    // Deleting files that were never written is a best-effort no-op, not an error.
+    DataIncrement data_increment(/*new_files=*/{CreateAppendDataFileMeta("abort-missing", 1)},
+                                 /*deleted_files=*/{}, /*changelog_files=*/{});
+    std::shared_ptr message =
+        std::make_shared(CreateIntRow(10), /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Abort({message}));
+
+    // A commit message that is not a CommitMessageImpl is rejected.
+    ASSERT_NOK_WITH_MSG(commit_impl->Abort({std::make_shared()}),
+                        "fail to cast commit message to impl");
+}
+
+TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) {
+    // A delete that fails with an IO error (e.g. permission denied) must be swallowed by
+    // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not
+    // propagate the failure.
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+
+    const BinaryRow partition = CreateIntRow(10);
+    const int32_t bucket = 0;
+    auto new_data_file = CreateAppendDataFileMeta("abort-io-fail-data", 1);
+    DataIncrement data_increment(/*new_files=*/{new_data_file}, /*deleted_files=*/{},
+                                 /*changelog_files=*/{});
+    std::shared_ptr message = std::make_shared(
+        partition, bucket, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {}));
+
+    // Materialize the file so we can observe that a failed delete leaves it untouched.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr data_pf,
+                         commit_impl->path_factory_->CreateDataFilePathFactory(partition, bucket));
+    const std::string data_path = data_pf->ToPath(new_data_file);
+    ASSERT_OK(file_system_->WriteFile(data_path, /*content=*/"", /*overwrite=*/false));
+
+    // Fault-inject an IO error on the delete. Abort performs no local file IO before its delete
+    // loop, so position 0 lands on the delete itself. The error must be swallowed. Clearing on
+    // scope exit keeps the process-global hook from leaking into other tests.
+    auto io_hook = IOHook::GetInstance();
+    ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+    io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR);
+    ASSERT_OK(commit_impl->Abort({message}));
+    io_hook->Clear();
+
+    // Because the delete failed, the file remains on disk.
+    ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(data_path));
+    ASSERT_TRUE(exist);
+}
+
+TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) {
+    // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot
+    // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still
+    // materialized.
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .IgnoreEmptyCommit(true)
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+
+    ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(table_path_ + "/snapshot/snapshot-1"));
+    ASSERT_TRUE(exist);
+    auto commit_impl = dynamic_cast(commit.get());
+    ASSERT_TRUE(commit_impl);
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.GetCommitKind());
+    std::vector manifests;
+    ASSERT_OK(commit_impl->manifest_list_->ReadDeltaManifests(snapshot, &manifests));
+    for (const auto& manifest : manifests) {
+        ASSERT_EQ(0, manifest.NumAddedFiles());
+        ASSERT_EQ(0, manifest.NumDeletedFiles());
+    }
+}
+
+TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .IgnoreEmptyCommit(true)
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    std::vector> msgs;
+    auto committable = commit_impl->CreateManifestCommittable(1, msgs, 30);
+    ASSERT_TRUE(committable);
+    EXPECT_EQ(1, committable->Identifier());
+    EXPECT_EQ(30, committable->Watermark().value());
+    ASSERT_TRUE(IsEqualMsgs(msgs, committable->FileCommittables()));
+}
+
+TEST_F(FileStoreCommitImplTest, TestCollectChanges) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::BUCKET, "10")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/"
+                              "commit_messages-01",
+                          /*version=*/3);
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    ASSERT_OK_AND_ASSIGN(ManifestEntryChanges changes, commit_impl->CollectChanges(msgs));
+    ASSERT_EQ(changes.append_table_files.size(), 3u);
+    ASSERT_EQ(changes.append_changelog.size(), 0u);
+    ASSERT_EQ(changes.compact_table_files.size(), 0u);
+    ASSERT_EQ(changes.compact_changelog.size(), 0u);
+    ASSERT_EQ(changes.append_index_files.size(), 0u);
+    ASSERT_EQ(changes.compact_index_files.size(), 0u);
+    ASSERT_EQ(changes.append_table_files[0].Kind(), FileKind::Add());
+    ASSERT_EQ(changes.append_table_files[0].Bucket(), 0);
+    ASSERT_EQ(changes.append_table_files[0].TotalBuckets(), 10);
+    ASSERT_EQ(changes.append_table_files[0].Level(), 0);
+    ASSERT_EQ(changes.append_table_files[0].FileName(),
+              "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc");
+    ASSERT_EQ(changes.append_table_files[1].Kind(), FileKind::Add());
+    ASSERT_EQ(changes.append_table_files[1].Bucket(), 1);
+    ASSERT_EQ(changes.append_table_files[1].TotalBuckets(), 10);
+    ASSERT_EQ(changes.append_table_files[1].Level(), 0);
+    ASSERT_EQ(changes.append_table_files[1].FileName(),
+              "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc");
+    ASSERT_EQ(changes.append_table_files[2].Kind(), FileKind::Add());
+    ASSERT_EQ(changes.append_table_files[2].Bucket(), 0);
+    ASSERT_EQ(changes.append_table_files[2].TotalBuckets(), 10);
+    ASSERT_EQ(changes.append_table_files[2].Level(), 0);
+    ASSERT_EQ(changes.append_table_files[2].FileName(),
+              "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc");
+}
+
+TEST_F(FileStoreCommitImplTest, TestFilterCommitted) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/"
+                              "commit_messages-01",
+                          /*version=*/3);
+    auto committable = commit_impl->CreateManifestCommittable(1, msgs, std::nullopt);
+    std::vector> committables = {committable};
+
+    // Test with no previous snapshots
+    ASSERT_OK_AND_ASSIGN(auto filtered_committables, commit_impl->FilterCommitted(committables));
+    ASSERT_EQ(filtered_committables.size(), committables.size());
+
+    // Test with a previous snapshot
+    ASSERT_OK(commit_impl->Commit(committable, /*check_append_files=*/false));
+    ASSERT_OK_AND_ASSIGN(filtered_committables, commit_impl->FilterCommitted(committables));
+    ASSERT_EQ(filtered_committables.size(), 0);
+}
+
+TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs1 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/"
+                              "commit_messages-01",
+                          /*version=*/3);
     auto committable1 = commit_impl->CreateManifestCommittable(1, msgs1, std::nullopt);
 
     std::vector> msgs2 =
@@ -1374,6 +1815,37 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) {
     ASSERT_EQ(filtered_committables[0]->Identifier(), committable2->Identifier());
 }
 
+TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs1 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/"
+                              "commit_messages-01",
+                          /*version=*/3);
+    std::vector> msgs2 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/"
+                              "commit_messages-02",
+                          /*version=*/3);
+
+    auto committable1 = commit_impl->CreateManifestCommittable(1, msgs1, std::nullopt);
+    auto committable2 = commit_impl->CreateManifestCommittable(1, msgs2, std::nullopt);
+
+    std::vector> committables = {committable1, committable2};
+    ASSERT_NOK_WITH_MSG(commit_impl->FilterCommitted(committables),
+                        "Committables must be sorted according to identifiers before filtering");
+}
+
 TEST_F(FileStoreCommitImplTest, FilterAndCommit) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -1445,6 +1917,75 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) {
     ASSERT_NOK(commit_impl->FilterAndCommit(inputs1));
 }
 
+TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    const BinaryRow partition = CreateIntRow(10);
+    const std::string base_name = "compacted-changelog-8e049c65-5ce4-4ce7-b1b0-78ce694ab351";
+    const std::string fake_name = base_name + "$0-39253-39253-35699.cc-parquet";
+    const std::string real_name = base_name + "$0-39253.cc-parquet";
+
+    ASSERT_OK(PrepareFakeFiles({"/f1=10/bucket-0/" + real_name}));
+
+    auto fake_changelog_file = CreateAppendDataFileMeta(fake_name, /*row_count=*/1);
+    DataIncrement data_increment(
+        /*new_files=*/{}, /*deleted_files=*/{},
+        /*changelog_files=*/{fake_changelog_file}, /*new_index_files=*/{},
+        /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{},
+                                       /*changelog_files=*/{});
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/1, /*total_buckets=*/2, data_increment, compact_increment);
+
+    std::map>> inputs;
+    inputs[1] = {message};
+    ASSERT_OK_AND_ASSIGN(int32_t actual_committed, commit_impl->FilterAndCommit(inputs));
+    ASSERT_EQ(1u, actual_committed);
+}
+
+TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    const BinaryRow partition = CreateIntRow(10);
+    const std::string compact_before_name = "missing-compact-before.orc";
+    const std::string compact_after_name = "existing-compact-after.orc";
+    ASSERT_OK(PrepareFakeFiles({"/f1=10/bucket-0/" + compact_after_name}));
+
+    auto compact_before_file = CreateAppendDataFileMeta(compact_before_name, /*row_count=*/1);
+    auto compact_after_file = CreateAppendDataFileMeta(compact_after_name, /*row_count=*/1);
+    DataIncrement data_increment(
+        /*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{}, /*new_index_files=*/{},
+        /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(/*compact_before=*/{compact_before_file},
+                                       /*compact_after=*/{compact_after_file},
+                                       /*changelog_files=*/{});
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    std::map>> inputs;
+    inputs[1] = {message};
+    ASSERT_OK_AND_ASSIGN(int32_t actual_committed, commit_impl->FilterAndCommit(inputs));
+    ASSERT_EQ(1u, actual_committed);
+}
+
 TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -1485,6 +2026,215 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) {
     ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc"));
 }
 
+TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> new_index_files;
+    new_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-1"));
+    DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {});
+    std::shared_ptr msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {}));
+
+    ASSERT_OK(commit_impl->Commit({msg}, 1));
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind());
+    ASSERT_TRUE(snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_EQ(1u, index_entries.size());
+    ASSERT_EQ("bitmap-index-commit-1", index_entries[0].index_file->FileName());
+}
+
+TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::ROW_TRACKING_ENABLED, "true")
+                             .AddOption(Options::DATA_EVOLUTION_ENABLED, "true")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    DataIncrement data_increment_1({CreateAppendDataFileMeta("data-with-row-id", 10)}, {}, {});
+    std::shared_ptr msg_1 =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment_1, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Commit({msg_1}, 1));
+
+    std::vector> global_index_files;
+    global_index_files.push_back(CreateGlobalIndexFileMeta("global-index-out-of-range",
+                                                           /*row_range_start=*/0,
+                                                           /*row_range_end=*/10));
+    DataIncrement data_increment_2({}, {}, {}, std::move(global_index_files), {});
+    std::shared_ptr msg_2 =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment_2, CompactIncrement({}, {}, {}));
+
+    ASSERT_NOK_WITH_MSG(commit_impl->Commit({msg_2}, 2), "Global index row ID existence conflict");
+}
+
+TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .IgnoreEmptyCommit(true)
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> compact_index_files;
+    compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-compact-1"));
+    DataIncrement data_increment({}, {}, {});
+    CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {});
+    std::shared_ptr msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    ASSERT_OK(commit_impl->Commit({msg}, 1));
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind());
+    ASSERT_TRUE(snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_EQ(1u, index_entries.size());
+    ASSERT_EQ("bitmap-index-commit-compact-1", index_entries[0].index_file->FileName());
+}
+
+TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> new_index_files;
+    new_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1"));
+    DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {});
+    std::shared_ptr msg_1 =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment_1, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Commit({msg_1}, 1));
+
+    std::vector> deleted_index_files;
+    deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1"));
+    DataIncrement data_increment_2({}, {}, {}, {}, std::move(deleted_index_files));
+    std::shared_ptr msg_2 =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment_2, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Commit({msg_2}, 2));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_TRUE(snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_TRUE(index_entries.empty());
+}
+
+TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .IgnoreEmptyCommit(true)
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> new_index_files;
+    new_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1"));
+    DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {});
+    std::shared_ptr msg_1 =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2,
+                                            data_increment_1, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Commit({msg_1}, 1));
+
+    std::vector> deleted_index_files;
+    deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1"));
+    DataIncrement data_increment_2({}, {}, {});
+    CompactIncrement compact_increment({}, {}, {}, {}, std::move(deleted_index_files));
+    std::shared_ptr msg_2 = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment_2, compact_increment);
+    ASSERT_OK(commit_impl->Commit({msg_2}, 2));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind());
+    ASSERT_TRUE(snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_TRUE(index_entries.empty());
+}
+
+TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> compact_index_files;
+    compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-1"));
+    DataIncrement data_increment({}, {}, {});
+    CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {});
+    std::shared_ptr msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    ASSERT_OK(commit_impl->Overwrite({}, {msg}, 1));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot,
+                         commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
+    ASSERT_TRUE(compact_snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_EQ(1u, index_entries.size());
+    ASSERT_EQ("bitmap-index-compact-1", index_entries[0].index_file->FileName());
+}
+
 TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
@@ -1541,7 +2291,46 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) {
     ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc"));
 }
 
-TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) {
+TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    std::vector> compact_index_files;
+    compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-filter-compact-1"));
+    DataIncrement data_increment({}, {}, {});
+    CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {});
+    std::shared_ptr msg = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    ASSERT_OK_AND_ASSIGN(int32_t actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 10));
+    ASSERT_EQ(1, actual_commit);
+    ASSERT_OK_AND_ASSIGN(actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 5));
+    ASSERT_EQ(0, actual_commit);
+
+    ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot,
+                         commit_impl->snapshot_manager_->LoadSnapshot(1));
+    ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind());
+    ASSERT_TRUE(compact_snapshot.Watermark());
+    ASSERT_EQ(10, compact_snapshot.Watermark().value());
+    ASSERT_TRUE(compact_snapshot.IndexManifest());
+
+    std::vector index_entries;
+    ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(),
+                                                      /*filter=*/nullptr, &index_entries));
+    ASSERT_EQ(1u, index_entries.size());
+    ASSERT_EQ("bitmap-index-filter-compact-1", index_entries[0].index_file->FileName());
+}
+
+TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) {
     CommitContextBuilder context_builder(table_path_, "commit_user_1");
     ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
                          context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
@@ -1567,14 +2356,13 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) {
 
     std::map partitions;
     partitions["f1"] = "10";
-    ASSERT_OK(commit_impl->Overwrite({partitions}, msgs2, 2));
+    ASSERT_OK(commit_impl->Overwrite(partitions, msgs2, 2));
     ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot());
     ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {}));
-    ASSERT_EQ(3u, entries1.size());
+    ASSERT_EQ(2u, entries1.size());
     std::set file_names = CollectFileNames(entries1);
     ASSERT_TRUE(IsStringInSet(file_names, "data-fd1d2255-43f2-4534-b4cc-08b29e662940-0.orc"));
     ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc"));
-    ASSERT_TRUE(IsStringInSet(file_names, "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"));
 }
 
 TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) {
@@ -1603,11 +2391,42 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) {
     ASSERT_TRUE(IsStringInSet(file_names, "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc"));
     ASSERT_TRUE(IsStringInSet(file_names, "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"));
 
-    // same file delete, then add, file will also be removed in result
-    ASSERT_OK(commit_impl->Overwrite({}, msgs1, 2));
+    // Java parity: overwrite provider adds DELETE(old) + ADD(newChanges) without dedup,
+    // so same-file overwrite fails on duplicate add.
+    ASSERT_NOK_WITH_MSG(commit_impl->Overwrite({}, msgs1, 2), "Trying to add file");
+}
+
+TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true")
+                             .Finish());
+
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+
+    ASSERT_OK(commit_impl->Commit(msgs, 1));
+    ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {}));
+    std::set file_names_before = CollectFileNames(entries1);
+    ASSERT_FALSE(file_names_before.empty());
+
+    // Committing exactly the same append files should be accepted and filtered out when
+    // commit.discard-duplicate-files=true.
+    ASSERT_OK(commit_impl->Commit(msgs, 2));
     ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot());
     ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {}));
-    ASSERT_EQ(0u, entries2.size());
+    std::set file_names_after = CollectFileNames(entries2);
+    ASSERT_EQ(file_names_before, file_names_after);
 }
 
 TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) {
@@ -1663,4 +2482,502 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) {
     ASSERT_TRUE(commit_run_complete);
 }
 
+TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) {
+    // Verify: the object store check in FileStoreCommit::Create is skipped when
+    // UseRESTCatalogCommit is true. We can't use an actual oss:// path in unit
+    // tests (LocalFileSystem rejects the scheme), but we verify the condition
+    // by confirming that the "enable-object-store-commit-in-inte-test" flag is
+    // not needed when UseRESTCatalogCommit is enabled. End-to-end oss:// testing
+    // is covered by duckdb-paimon integration tests.
+    ASSERT_OK_AND_ASSIGN(bool is_oss, FileSystem::IsObjectStore("oss://bucket/path"));
+    ASSERT_TRUE(is_oss);
+
+    // REST commit with local path should work without the object store flag
+    CommitContextBuilder builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(
+        auto ctx,
+        builder.AddOption(Options::MANIFEST_FORMAT, "orc").UseRESTCatalogCommit(true).Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx)));
+
+    auto msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          3);
+    ASSERT_GT(msgs.size(), 0);
+    ASSERT_OK(commit->Commit(msgs));
+    ASSERT_OK_AND_ASSIGN(auto json, commit->GetLastCommitTableRequest());
+    ASSERT_FALSE(json.empty());
+}
+
+TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) {
+    auto pk_dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(pk_dir);
+    std::string pk_root = pk_dir->Str();
+    ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {}));
+    ASSERT_OK(catalog->CreateDatabase("db", {}, false));
+
+    arrow::Schema pk_schema(
+        {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())});
+    ::ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok());
+    std::map table_options = {{Options::BUCKET, "4"}};
+    ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema,
+                                   /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options,
+                                   /*ignore_if_exists=*/false));
+
+    std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl");
+
+    CommitContextBuilder builder(pk_table_path, "test_user");
+    builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true);
+    ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_TRUE(committer != nullptr);
+}
+
+TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsRejectsUnsupportedOptions) {
+    const std::vector unsupported_keys = {
+        "commit.strict-mode.last-safe-snapshot", "manifest.delete-file-drop-stats",
+        "sequence.snapshot-ordering", "pk-clustering-override"};
+    for (const auto& key : unsupported_keys) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{key, "true"}}));
+        ASSERT_NOK_WITH_MSG(FileStoreCommitImpl::ValidateCommitOptions(options),
+                            "not supported by C++ commit path");
+    }
+
+    // Supported options should validate successfully.
+    ASSERT_OK_AND_ASSIGN(CoreOptions ok_options,
+                         CoreOptions::FromMap({{Options::FILE_SYSTEM, "local"}}));
+    ASSERT_OK(FileStoreCommitImpl::ValidateCommitOptions(ok_options));
+}
+
+TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1),
+                        "partitions list cannot be empty");
+}
+
+TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    // Empty input map takes the FilterCommitted fast-exit path (nothing to commit).
+    ASSERT_OK_AND_ASSIGN(int32_t committed_empty, commit_impl->FilterAndCommit({}, 1));
+    ASSERT_EQ(0, committed_empty);
+
+    std::vector data_files = {
+        "/f1=10/bucket-0/data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc",
+        "/f1=10/bucket-1/data-6828284c-e707-49b5-af6b-69be79af120c-0.orc",
+        "/f1=20/bucket-0/data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc",
+        "/f1=10/bucket-1/data-fd1d2255-43f2-4534-b4cc-08b29e662940-0.orc",
+        "/f1=20/bucket-0/data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc"};
+    ASSERT_OK(PrepareFakeFiles(data_files));
+
+    std::vector> msgs1 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    std::vector> msgs2 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-02",
+                          /*version=*/3);
+
+    // Two identifiers in a single call exercises the sort-by-identifier comparator.
+    std::map>> inputs;
+    inputs[2] = msgs2;
+    inputs[1] = msgs1;
+    ASSERT_OK_AND_ASSIGN(int32_t committed, commit_impl->FilterAndCommit(inputs, 5));
+    ASSERT_EQ(2, committed);
+}
+
+TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    auto committable = std::make_shared(/*identifier=*/1);
+    committable->AddFileCommittable(std::make_shared());
+    ASSERT_NOK_WITH_MSG(commit_impl->CheckFilesExistence({committable}),
+                        "fail to cast commit message to impl");
+}
+
+TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    // New index files from both the data increment and the compact increment must be
+    // collected for the existence check.
+    DataIncrement data_increment(/*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{},
+                                 /*new_index_files=*/{CreateIndexFileMeta("new-index-missing")},
+                                 /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(
+        /*compact_before=*/{}, /*compact_after=*/{}, /*changelog_files=*/{},
+        /*new_index_files=*/{CreateIndexFileMeta("compact-index-missing")},
+        /*deleted_index_files=*/{});
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    auto committable = std::make_shared(/*identifier=*/1);
+    committable->AddFileCommittable(message);
+    // The referenced index files were never written, so the existence check reports them missing.
+    ASSERT_NOK_WITH_MSG(commit_impl->CheckFilesExistence({committable}), "have been deleted");
+}
+
+TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    // Files whose partition matches the overwrite target are accepted.
+    DataIncrement matching_increment({CreateAppendDataFileMeta("static-f1-10", 1)}, {}, {});
+    std::shared_ptr matching_msg =
+        std::make_shared(CreateIntRow(10), /*bucket=*/0, /*total_buckets=*/2,
+                                            matching_increment, CompactIncrement({}, {}, {}));
+    ASSERT_OK(commit_impl->Overwrite({{"f1", "10"}}, {matching_msg}, /*commit_identifier=*/1));
+
+    // Files belonging to a different partition than the overwrite target are rejected.
+    DataIncrement mismatching_increment({CreateAppendDataFileMeta("static-f1-20", 1)}, {}, {});
+    std::shared_ptr mismatching_msg =
+        std::make_shared(CreateIntRow(20), /*bucket=*/0, /*total_buckets=*/2,
+                                            mismatching_increment, CompactIncrement({}, {}, {}));
+    ASSERT_NOK_WITH_MSG(
+        commit_impl->Overwrite({{"f1", "10"}}, {mismatching_msg}, /*commit_identifier=*/2),
+        "does not belong to this partition");
+}
+
+TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+    const BinaryRow partition = CreateIntRow(10);
+
+    // Overwrite ignores changelog files but emits a warning listing them.
+    DataIncrement data_increment(
+        /*new_files=*/{CreateAppendDataFileMeta("overwrite-with-changelog", 1)},
+        /*deleted_files=*/{},
+        /*changelog_files=*/{CreateAppendDataFileMeta("ignored-append-changelog", 1)},
+        /*new_index_files=*/{}, /*deleted_index_files=*/{});
+    CompactIncrement compact_increment(
+        /*compact_before=*/{}, /*compact_after=*/{},
+        /*changelog_files=*/{CreateAppendDataFileMeta("ignored-compact-changelog", 1)});
+    std::shared_ptr message = std::make_shared(
+        partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment);
+
+    ASSERT_OK(commit_impl->Overwrite({}, {message}, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(std::optional snapshot,
+                         commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_TRUE(snapshot.has_value());
+}
+
+TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) {
+    auto pk_dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(pk_dir);
+    std::string pk_root = pk_dir->Str();
+    ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {}));
+    ASSERT_OK(catalog->CreateDatabase("db", {}, false));
+
+    arrow::Schema pk_schema(
+        {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())});
+    ::ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok());
+    std::map table_options = {{Options::BUCKET, "4"}};
+    ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema,
+                                   /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options,
+                                   /*ignore_if_exists=*/false));
+    std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl");
+
+    CommitContextBuilder builder(pk_table_path, "test_user");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::OVERWRITE_UPGRADE, "true")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    const BinaryRow partition = BinaryRow::EmptyRow();
+    // Bucket 0: non-overlapping key ranges -> files are upgraded to a higher level.
+    DataIncrement bucket0_increment(
+        {CreateLeveledDataFileMeta("pk-b0-lo", CreateIntRow(0), CreateIntRow(10), /*level=*/0),
+         CreateLeveledDataFileMeta("pk-b0-hi", CreateIntRow(20), CreateIntRow(30), /*level=*/0)},
+        {}, {});
+    std::shared_ptr bucket0_msg =
+        std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/4,
+                                            bucket0_increment, CompactIncrement({}, {}, {}));
+    // Bucket 1: overlapping key ranges -> files are kept at their original level.
+    DataIncrement bucket1_increment(
+        {CreateLeveledDataFileMeta("pk-b1-a", CreateIntRow(0), CreateIntRow(20), /*level=*/0),
+         CreateLeveledDataFileMeta("pk-b1-b", CreateIntRow(10), CreateIntRow(30), /*level=*/0)},
+        {}, {});
+    std::shared_ptr bucket1_msg =
+        std::make_shared(partition, /*bucket=*/1, /*total_buckets=*/4,
+                                            bucket1_increment, CompactIncrement({}, {}, {}));
+
+    ASSERT_OK(commit_impl->Overwrite({}, {bucket0_msg, bucket1_msg}, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::optional snapshot,
+                         commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_TRUE(snapshot.has_value());
+    ASSERT_OK_AND_ASSIGN(std::vector entries,
+                         commit_impl->GetAllFiles(snapshot.value(), {}));
+    ASSERT_EQ(4u, entries.size());
+
+    int32_t max_level = 0;
+    int32_t level0_count = 0;
+    for (const auto& entry : entries) {
+        max_level = std::max(max_level, entry.Level());
+        if (entry.Level() == 0) {
+            level0_count++;
+        }
+    }
+    // Bucket 0 files were upgraded above level 0, bucket 1 files stayed at level 0.
+    ASSERT_GT(max_level, 0);
+    ASSERT_EQ(2, level0_count);
+}
+
+TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AppendCommitCheckConflict(true)
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    ASSERT_GT(msgs.size(), 0);
+    ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1));
+    ASSERT_OK_AND_ASSIGN(
+        bool exist, file_system_->Exists(PathUtil::JoinPath(table_path_, "snapshot/snapshot-1")));
+    ASSERT_TRUE(exist);
+}
+
+TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) {
+    // First snapshot is committed in the default (scan) mode, so it does NOT carry the
+    // max-sequence-number property.
+    {
+        CommitContextBuilder context_builder(table_path_, "commit_user_1");
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                             context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                                 .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                                 .AddOption(Options::FILE_SYSTEM, "local")
+                                 .Finish());
+        ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+        std::vector> msgs1 =
+            GetCommitMessages(paimon::test::GetDataDir() +
+                                  "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                              /*version=*/3);
+        ASSERT_OK(commit->Commit(msgs1, /*commit_identifier=*/1));
+    }
+
+    // Second commit uses snapshot-init mode; since the previous snapshot lacks the property,
+    // the max sequence number is recomputed by scanning the base manifests.
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1));
+    if (snapshot1.Properties()) {
+        ASSERT_EQ(snapshot1.Properties().value().end(),
+                  snapshot1.Properties().value().find("sequence.generation.max-sequence-number"));
+    }
+
+    std::vector> msgs2 =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-02",
+                          /*version=*/3);
+    ASSERT_OK(commit_impl->Commit(msgs2, /*commit_identifier=*/2));
+
+    ASSERT_OK_AND_ASSIGN(Snapshot snapshot2, commit_impl->snapshot_manager_->LoadSnapshot(2));
+    ASSERT_TRUE(snapshot2.Properties());
+    auto iter = snapshot2.Properties().value().find("sequence.generation.max-sequence-number");
+    ASSERT_TRUE(iter != snapshot2.Properties().value().end());
+}
+
+TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    // A non-empty partition spec is forwarded to the overwrite so the partition list is populated.
+    std::map partition_spec = {{"f1", "10"}};
+    ASSERT_OK_AND_ASSIGN(int32_t actual_commit, commit_impl->FilterAndOverwrite(
+                                                    partition_spec, msgs, /*commit_identifier=*/1,
+                                                    /*watermark=*/10));
+    ASSERT_EQ(1, actual_commit);
+
+    ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.value().GetCommitKind());
+}
+
+TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::OVERWRITE_UPGRADE, "false")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector entries = {
+        CreateManifestEntry("upgrade-disabled-1", FileKind::Add())};
+    // overwrite-upgrade disabled: TryUpgrade returns the input unchanged.
+    ASSERT_OK_AND_ASSIGN(std::vector result, commit_impl->TryUpgrade(entries));
+    ASSERT_EQ(entries.size(), result.size());
+}
+
+TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) {
+    auto pk_dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(pk_dir);
+    std::string pk_root = pk_dir->Str();
+    ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {}));
+    ASSERT_OK(catalog->CreateDatabase("db", {}, false));
+
+    arrow::Schema pk_schema(
+        {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())});
+    ::ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok());
+    std::map table_options = {{Options::BUCKET, "4"}};
+    ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema,
+                                   /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options,
+                                   /*ignore_if_exists=*/false));
+    std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl");
+
+    CommitContextBuilder builder(pk_table_path, "test_user");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .AddOption(Options::OVERWRITE_UPGRADE, "true")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    // A PK-table entry already at a level above 0 bypasses the upgrade and returns the input.
+    std::vector entries = {
+        CreateManifestEntry("already-upgraded", BinaryRow::EmptyRow(), FileKind::Add(),
+                            DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), /*level=*/2,
+                            /*bucket=*/0)};
+    ASSERT_OK_AND_ASSIGN(std::vector result, commit_impl->TryUpgrade(entries));
+    ASSERT_EQ(entries.size(), result.size());
+    ASSERT_EQ(2, result[0].Level());
+}
+
+TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    std::vector> msgs =
+        GetCommitMessages(paimon::test::GetDataDir() +
+                              "/orc/append_09.db/append_09/commit_messages/commit_messages-01",
+                          /*version=*/3);
+    ASSERT_OK(commit_impl->Commit(msgs, /*commit_identifier=*/1));
+
+    ASSERT_OK_AND_ASSIGN(std::optional snapshot,
+                         commit_impl->snapshot_manager_->LatestSnapshot());
+    ASSERT_TRUE(snapshot);
+    // No delta entries -> no buckets to verify -> returns OK without scanning the snapshot.
+    ASSERT_OK(commit_impl->CheckSameBucketFromSnapshot(/*delta_entries=*/{}, snapshot));
+}
+
+TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    // No manifests to scan -> no sequence number found -> returns nullopt.
+    ASSERT_OK_AND_ASSIGN(std::optional max_seq, commit_impl->MaxSequenceNumber({}));
+    ASSERT_FALSE(max_seq.has_value());
+}
+
+TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) {
+    CommitContextBuilder context_builder(table_path_, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+    auto commit_impl = std::dynamic_pointer_cast(
+        std::shared_ptr(std::move(commit)));
+
+    ASSERT_FALSE(commit_impl->conflict_detection_.HasRowIdCheckFromSnapshot());
+    // RowIdCheckConflict records the snapshot to verify against and returns *this for chaining.
+    FileStoreCommit& returned = commit_impl->RowIdCheckConflict(/*row_id_check_from_snapshot=*/5);
+    ASSERT_EQ(commit_impl.get(), &returned);
+    ASSERT_TRUE(commit_impl->conflict_detection_.HasRowIdCheckFromSnapshot());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp
index 3ee13c9a..b43e3215 100644
--- a/src/paimon/core/operation/file_store_commit_test.cpp
+++ b/src/paimon/core/operation/file_store_commit_test.cpp
@@ -29,10 +29,21 @@
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/commit_context.h"
+#include "paimon/common/utils/linked_hash_map.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/compact_increment.h"
+#include "paimon/core/io/data_increment.h"
 #include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/defs.h"
+#include "paimon/fs/local/local_file_system.h"
 #include "paimon/result.h"
+#include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -75,4 +86,62 @@ TEST(FileStoreCommitTest, TestCreate) {
     ASSERT_TRUE(commit_impl);
 }
 
+TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) {
+    auto string_field = arrow::field("f0", arrow::utf8());
+    auto int_field = arrow::field("f1", arrow::int32());
+    auto int_field1 = arrow::field("f2", arrow::int32());
+    auto double_field = arrow::field("f3", arrow::float64());
+    auto schema =
+        arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field}));
+
+    ::ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok());
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+
+    std::map options = {{Options::FILE_FORMAT, "orc"},
+                                                  {Options::TARGET_FILE_SIZE, "1024"},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::BUCKET, "2"},
+                                                  {Options::BUCKET_KEY, "f2"}};
+
+    ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options));
+    ASSERT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false));
+    ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &arrow_schema,
+                                   /*partition_keys=*/{"f1"},
+                                   /*primary_keys=*/{}, options,
+                                   /*ignore_if_exists=*/false));
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    CommitContextBuilder context_builder(table_path, "commit_user");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         context_builder.AddOption(Options::MANIFEST_FORMAT, "orc")
+                             .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb")
+                             .AddOption(Options::FILE_SYSTEM, "local")
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context)));
+
+    LinkedHashMap dv_ranges;
+    dv_ranges.insert_or_assign("data-file-1",
+                               DeletionVectorMeta(
+                                   /*data_file_name=*/"data-file-1", /*offset=*/0, /*length=*/10,
+                                   /*cardinality=*/1));
+    std::vector> new_index_files;
+    new_index_files.push_back(std::make_shared(
+        DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-index-1", 100, 1,
+        /*dv_ranges=*/dv_ranges, /*external_path=*/std::nullopt));
+    DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {});
+    std::shared_ptr msg = std::make_shared(
+        BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()), /*bucket=*/0,
+        /*total_bucket=*/2, data_increment, CompactIncrement({}, {}, {}));
+
+    ASSERT_OK(commit->Commit({msg}, /*commit_identifier=*/1));
+
+    auto fs = std::make_shared();
+    SnapshotManager snapshot_manager(fs, table_path);
+    ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot());
+    ASSERT_TRUE(snapshot);
+    ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.value().GetCommitKind());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp
index 65acb7a6..681e772e 100644
--- a/src/paimon/core/operation/file_store_scan.cpp
+++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -18,6 +18,7 @@
 
 #include "paimon/core/operation/file_store_scan.h"
 
+#include 
 #include 
 #include 
 #include 
@@ -28,7 +29,9 @@
 
 #include "arrow/type.h"
 #include "fmt/format.h"
+#include "paimon/cache/cache.h"
 #include "paimon/common/data/binary_array.h"
+#include "paimon/common/data/blob_utils.h"
 #include "paimon/common/executor/future.h"
 #include "paimon/common/predicate/literal_converter.h"
 #include "paimon/common/types/data_field.h"
@@ -39,13 +42,17 @@
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/manifest/snapshot_live_manifest_entries.h"
 #include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/partition/partition_info.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/stats/simple_stats_evolution.h"
+#include "paimon/core/utils/branch_manager.h"
 #include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_segment.h"
 #include "paimon/predicate/literal.h"
 #include "paimon/predicate/predicate.h"
 #include "paimon/predicate/predicate_builder.h"
@@ -71,11 +78,12 @@ Result> FileStoreScan::ReconstructPredicateWithNonCas
                 fmt::format("field {} in predicate is not included in table schema", field_name));
         }
         auto data_iter = id_to_data_fields.find(table_iter->second.Id());
-        if (data_iter != id_to_data_fields.end()) {
-            // Exclude fields requiring casting to avoid false negatives in stats filtering.
-            if (!data_iter->second.second.Type()->Equals(table_iter->second.Type())) {
-                excluded_field_names.insert(field_name);
-            }
+        if (data_iter == id_to_data_fields.end()) {
+            continue;
+        }
+        // Exclude fields requiring casting to avoid false negatives in stats filtering.
+        if (!data_iter->second.second.Type()->Equals(table_iter->second.Type())) {
+            excluded_field_names.insert(field_name);
         }
     }
     return PredicateUtils::ExcludePredicateWithFields(predicate, excluded_field_names);
@@ -111,7 +119,8 @@ Result> FileStoreScan::ReadPartitionEntries() const
     PAIMON_RETURN_NOT_OK(
         ReadManifests(&snapshot, &all_manifest_file_metas, &filtered_manifest_file_metas));
     std::vector manifest_entries;
-    PAIMON_RETURN_NOT_OK(ReadFileEntries(filtered_manifest_file_metas, &manifest_entries));
+    PAIMON_RETURN_NOT_OK(ReadFileEntries(filtered_manifest_file_metas, &manifest_entries,
+                                         /*apply_scan_filter=*/true));
     std::unordered_map partitions;
     PAIMON_RETURN_NOT_OK(PartitionEntry::Merge(manifest_entries, &partitions));
 
@@ -134,7 +143,26 @@ Result> FileStoreScan::CreatePlan() cons
         ReadManifests(&snapshot, &all_manifest_file_metas, &filtered_manifest_file_metas));
 
     std::vector manifest_entries;
-    PAIMON_RETURN_NOT_OK(ReadManifestEntries(filtered_manifest_file_metas, &manifest_entries));
+    const bool use_snapshot_live_manifest_cache =
+        snapshot.has_value() && scan_mode_ == ScanMode::ALL &&
+        core_options_.GetScanManifestEntryCacheMaxSnapshots() > 0 &&
+        core_options_.GetCache() != nullptr && !table_path_.empty() &&
+        !row_range_index_.has_value() && bucket_filter_.has_value();
+    if (use_snapshot_live_manifest_cache) {
+        PAIMON_RETURN_NOT_OK(ReadManifestEntriesWithCache(
+            snapshot.value(), all_manifest_file_metas, bucket_filter_.value(), &manifest_entries));
+        std::vector filtered_entries;
+        filtered_entries.reserve(manifest_entries.size());
+        for (auto& entry : manifest_entries) {
+            PAIMON_ASSIGN_OR_RAISE(bool keep, FilterManifestEntry(entry));
+            if (keep) {
+                filtered_entries.emplace_back(std::move(entry));
+            }
+        }
+        manifest_entries = std::move(filtered_entries);
+    } else {
+        PAIMON_RETURN_NOT_OK(ReadManifestEntries(filtered_manifest_file_metas, &manifest_entries));
+    }
     PAIMON_ASSIGN_OR_RAISE(manifest_entries,
                            PostFilterManifestEntries(std::move(manifest_entries)));
 
@@ -169,9 +197,11 @@ Result> FileStoreScan::CreatePlan() cons
     metrics_->ObserveHistogram(ScanMetrics::SCAN_DURATION, static_cast(scan_duration_ms));
     metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID,
                          snapshot.has_value() ? snapshot.value().Id() : int64_t{0});
-    metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size());
-    metrics_->SetCounter(ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES,
-                         all_data_files - manifest_entries.size());
+    metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS,
+                         static_cast(filtered_manifest_file_metas.size()));
+    metrics_->SetCounter(
+        ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES,
+        std::max(int64_t{0}, all_data_files - static_cast(manifest_entries.size())));
     metrics_->SetCounter(ScanMetrics::LAST_SCAN_RESULTED_TABLE_FILES, manifest_entries.size());
     return std::make_shared(scan_mode_, snapshot,
                                                     std::move(manifest_entries));
@@ -217,12 +247,19 @@ Status FileStoreScan::ReadManifestsWithSnapshot(const Snapshot& snapshot,
 }
 
 Status FileStoreScan::ReadFileEntries(const std::vector& manifest_metas,
-                                      std::vector* manifest_entries) const {
+                                      std::vector* manifest_entries,
+                                      bool apply_scan_filter) const {
     std::vector>>> futures;
     for (const auto& meta : manifest_metas) {
-        auto read_meta_task = [this, &meta]() -> Result> {
+        auto read_meta_task = [this, meta,
+                               apply_scan_filter]() -> Result> {
             std::vector tmp_entries;
-            PAIMON_RETURN_NOT_OK(ReadManifestFileMeta(meta, &tmp_entries));
+            if (apply_scan_filter) {
+                PAIMON_RETURN_NOT_OK(ReadManifestFileMeta(meta, &tmp_entries));
+            } else {
+                PAIMON_RETURN_NOT_OK(
+                    manifest_file_->Read(meta.FileName(), /*filter=*/nullptr, &tmp_entries));
+            }
             return tmp_entries;
         };
         futures.push_back(Via(executor_.get(), read_meta_task));
@@ -250,29 +287,129 @@ Status FileStoreScan::ReadManifestEntries(const std::vector& m
     return ReadAndNoMergeFileEntries(manifest_metas, manifest_entries);
 }
 
-Status FileStoreScan::ReadAndMergeFileEntries(const std::vector& manifest_metas,
-                                              std::vector* merged_entries) const {
+// Cache merged live manifest entries for one bucket before applying scan filters. Each cache value
+// keeps a bounded number of snapshot results for the same table/branch/bucket. Exact snapshot hits
+// can be returned directly; cache misses rebuild the target snapshot bucket from the target
+// snapshot's data manifests.
+Status FileStoreScan::ReadManifestEntriesWithCache(
+    const Snapshot& snapshot, const std::vector& all_manifest_metas,
+    int32_t bucket, std::vector* manifest_entries) const {
+    PAIMON_ASSIGN_OR_RAISE(SnapshotLiveManifestEntries cached_entries,
+                           LoadSnapshotLiveManifestEntries(bucket));
+    std::optional cached =
+        cached_entries.LatestBeforeOrEqual(snapshot.Id());
+    if (cached && cached->snapshot_id == snapshot.Id()) {
+        *manifest_entries = *cached->entries;
+        return Status::OK();
+    }
+
+    // Rebuild the target snapshot bucket from all manifests and write the live entries back to the
+    // cache.
+    std::vector bucket_manifest_metas;
+    for (const auto& meta : all_manifest_metas) {
+        if (MayContainBucket(meta, bucket)) {
+            bucket_manifest_metas.push_back(meta);
+        }
+    }
+    PAIMON_RETURN_NOT_OK(
+        ReadAndMergeBucketFileEntries(bucket_manifest_metas, bucket, manifest_entries));
+    std::vector cache_entries = *manifest_entries;
+    cached_entries.Put(snapshot.Id(), std::move(cache_entries));
+    PAIMON_RETURN_NOT_OK(StoreSnapshotLiveManifestEntries(bucket, cached_entries));
+    return Status::OK();
+}
+
+std::shared_ptr FileStoreScan::SnapshotLiveManifestEntriesCacheKey(int32_t bucket) const {
+    return CacheKey::ForSnapshotLiveManifestEntries(
+        table_path_, BranchManager::NormalizeBranch(core_options_.GetBranch()), bucket);
+}
+
+Result FileStoreScan::LoadSnapshotLiveManifestEntries(
+    int32_t bucket) const {
+    auto supplier = [](const std::shared_ptr&) -> Result> {
+        return std::shared_ptr();
+    };
+    std::shared_ptr cache_key = SnapshotLiveManifestEntriesCacheKey(bucket);
+    const auto max_snapshots = core_options_.GetScanManifestEntryCacheMaxSnapshots();
+    Result> cache_result =
+        core_options_.GetCache()->Get(cache_key, supplier);
+    if (!cache_result.ok() || !cache_result.value()) {
+        return SnapshotLiveManifestEntries(max_snapshots);
+    }
+    Result deserialized = SnapshotLiveManifestEntries::Deserialize(
+        cache_result.value()->GetSegment(), max_snapshots, pool_);
+    if (!deserialized.ok()) {
+        return SnapshotLiveManifestEntries(max_snapshots);
+    }
+    return std::move(deserialized.value());
+}
+
+Status FileStoreScan::StoreSnapshotLiveManifestEntries(
+    int32_t bucket, const SnapshotLiveManifestEntries& entries) const {
+    Result> bytes_result = entries.Serialize(pool_);
+    if (!bytes_result.ok()) {
+        return Status::OK();
+    }
+    auto cache_value =
+        std::make_shared(MemorySegment::Wrap(bytes_result.value()), CacheCallback());
+    Status status =
+        core_options_.GetCache()->Put(SnapshotLiveManifestEntriesCacheKey(bucket), cache_value);
+    return status.ok() ? status : Status::OK();
+}
+
+Status FileStoreScan::ReadAndMergeBucketFileEntries(
+    const std::vector& manifest_metas, int32_t bucket,
+    std::vector* merged_entries) const {
     std::vector unmerged_entries;
-    PAIMON_RETURN_NOT_OK(ReadFileEntries(manifest_metas, &unmerged_entries));
+    std::vector entries;
+    PAIMON_RETURN_NOT_OK(ReadFileEntries(manifest_metas, &entries, /*apply_scan_filter=*/false));
+    unmerged_entries.reserve(entries.size());
+    for (auto& entry : entries) {
+        if (entry.Bucket() == bucket) {
+            unmerged_entries.emplace_back(std::move(entry));
+        }
+    }
+    return MergeLiveEntries(unmerged_entries, merged_entries);
+}
+
+Status FileStoreScan::MergeLiveEntries(const std::vector& unmerged_entries,
+                                       std::vector* live_entries) {
     std::unordered_set deleted_entries;
     for (const auto& entry : unmerged_entries) {
         if (entry.Kind() == FileKind::Delete()) {
             deleted_entries.insert(entry.CreateIdentifier());
         }
     }
-    for (auto& entry : unmerged_entries) {
+    for (const auto& entry : unmerged_entries) {
         if (entry.Kind() == FileKind::Add() &&
             deleted_entries.find(entry.CreateIdentifier()) == deleted_entries.end()) {
-            merged_entries->push_back(std::move(entry));
+            live_entries->push_back(entry);
         }
     }
     return Status::OK();
 }
 
+Status FileStoreScan::ReadAndMergeFileEntries(const std::vector& manifest_metas,
+                                              std::vector* merged_entries) const {
+    std::vector unmerged_entries;
+    PAIMON_RETURN_NOT_OK(
+        ReadFileEntries(manifest_metas, &unmerged_entries, /*apply_scan_filter=*/true));
+    return MergeLiveEntries(unmerged_entries, merged_entries);
+}
+
 Status FileStoreScan::ReadAndNoMergeFileEntries(
     const std::vector& manifest_metas,
     std::vector* manifest_entries) const {
-    return ReadFileEntries(manifest_metas, manifest_entries);
+    return ReadFileEntries(manifest_metas, manifest_entries, /*apply_scan_filter=*/true);
+}
+
+bool FileStoreScan::MayContainBucket(const ManifestFileMeta& manifest, int32_t bucket) const {
+    const std::optional& min_bucket = manifest.MinBucket();
+    const std::optional& max_bucket = manifest.MaxBucket();
+    if (min_bucket && max_bucket) {
+        return bucket >= min_bucket.value() && bucket <= max_bucket.value();
+    }
+    return true;
 }
 
 Result FileStoreScan::FilterManifestFileMeta(const ManifestFileMeta& manifest) const {
@@ -318,37 +455,38 @@ bool FileStoreScan::FilterManifestByRowRanges(const ManifestFileMeta& manifest)
 
 Status FileStoreScan::ReadManifestFileMeta(const ManifestFileMeta& manifest,
                                            std::vector* entries) const {
-    auto filter = [&](const ManifestEntry& entry) -> Result {
-        if (partition_filter_) {
-            PAIMON_ASSIGN_OR_RAISE(bool res,
-                                   partition_filter_->Test(partition_schema_, entry.Partition()));
-            if (!res) {
-                return false;
-            }
-        }
-        if (only_read_real_buckets_ && entry.Bucket() < 0) {
-            return false;
-        }
-        if (bucket_filter_ != std::nullopt && entry.Bucket() != bucket_filter_.value()) {
-            return false;
-        }
-        if (level_filter_ != nullptr && !level_filter_(entry.Level())) {
-            return false;
-        }
-        return true;
-    };
     std::vector unfiltered_entries;
-    PAIMON_RETURN_NOT_OK(manifest_file_->Read(manifest.FileName(), filter, &unfiltered_entries));
+    PAIMON_RETURN_NOT_OK(manifest_file_->Read(
+        manifest.FileName(),
+        [this](const ManifestEntry& entry) -> Result { return FilterManifestEntry(entry); },
+        &unfiltered_entries));
     entries->reserve(entries->size() + unfiltered_entries.size());
     for (auto& entry : unfiltered_entries) {
-        PAIMON_ASSIGN_OR_RAISE(bool res, FilterByStats(entry));
-        if (res) {
-            entries->emplace_back(std::move(entry));
-        }
+        entries->emplace_back(std::move(entry));
     }
     return Status::OK();
 }
 
+Result FileStoreScan::FilterManifestEntry(const ManifestEntry& entry) const {
+    if (partition_filter_) {
+        PAIMON_ASSIGN_OR_RAISE(bool res,
+                               partition_filter_->Test(partition_schema_, entry.Partition()));
+        if (!res) {
+            return false;
+        }
+    }
+    if (only_read_real_buckets_ && entry.Bucket() < 0) {
+        return false;
+    }
+    if (bucket_filter_ != std::nullopt && entry.Bucket() != bucket_filter_.value()) {
+        return false;
+    }
+    if (level_filter_ != nullptr && !level_filter_(entry.Level())) {
+        return false;
+    }
+    return FilterByStats(entry);
+}
+
 Status FileStoreScan::SplitAndSetFilter(const std::vector& partition_keys,
                                         const std::shared_ptr& arrow_schema,
                                         const std::shared_ptr& scan_filters) {
@@ -356,8 +494,12 @@ Status FileStoreScan::SplitAndSetFilter(const std::vector& partitio
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping_builder,
                                FieldMappingBuilder::Create(arrow_schema, partition_keys,
                                                            scan_filters->GetPredicate()));
+        PAIMON_ASSIGN_OR_RAISE(std::vector data_fields,
+                               DataField::ConvertArrowSchemaToDataFields(arrow_schema));
+        auto converted_fields = BlobUtils::ConvertBlobInlineDataFields(
+            data_fields, core_options_.GetBlobInlineFields());
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping,
-                               mapping_builder->CreateFieldMapping(arrow_schema));
+                               mapping_builder->CreateFieldMapping(converted_fields));
         if (mapping->partition_info != std::nullopt) {
             const auto& partition_info = mapping->partition_info.value();
             partition_schema_ =
diff --git a/src/paimon/core/operation/file_store_scan.h b/src/paimon/core/operation/file_store_scan.h
index 0361a9a6..53d7825a 100644
--- a/src/paimon/core/operation/file_store_scan.h
+++ b/src/paimon/core/operation/file_store_scan.h
@@ -23,7 +23,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -43,6 +42,7 @@
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/manifest/partition_entry.h"
+#include "paimon/core/manifest/snapshot_live_manifest_entries.h"
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/snapshot.h"
 #include "paimon/core/table/source/scan_mode.h"
@@ -59,6 +59,7 @@ class Schema;
 }  // namespace arrow
 
 namespace paimon {
+class CacheKey;
 class Executor;
 class FileKind;
 class ManifestFile;
@@ -122,6 +123,11 @@ class FileStoreScan {
         return this;
     }
 
+    FileStoreScan* WithTablePath(const std::string& table_path) {
+        table_path_ = table_path;
+        return this;
+    }
+
     virtual FileStoreScan* EnableValueFilter() {
         return this;
     }
@@ -239,6 +245,26 @@ class FileStoreScan {
     Status ReadManifestEntries(const std::vector& manifest_metas,
                                std::vector* manifest_entries) const;
 
+    Status ReadManifestEntriesWithCache(const Snapshot& snapshot,
+                                        const std::vector& bucket_manifest_metas,
+                                        int32_t bucket,
+                                        std::vector* manifest_entries) const;
+    std::shared_ptr SnapshotLiveManifestEntriesCacheKey(int32_t bucket) const;
+    Result LoadSnapshotLiveManifestEntries(int32_t bucket) const;
+    Status StoreSnapshotLiveManifestEntries(int32_t bucket,
+                                            const SnapshotLiveManifestEntries& entries) const;
+
+    Status ReadAndMergeBucketFileEntries(const std::vector& manifest_metas,
+                                         int32_t bucket,
+                                         std::vector* merged_entries) const;
+
+    /// Merge raw manifest entries into the set of currently-live files. Entries are deduplicated
+    /// by identifier (matching Add cancels a prior or following Delete), and lingering Delete
+    /// entries are dropped so the caller receives Add-only output, matching the semantics of
+    /// `ReadAndMergeFileEntries`.
+    static Status MergeLiveEntries(const std::vector& unmerged_entries,
+                                   std::vector* live_entries);
+
     Status ReadAndMergeFileEntries(const std::vector& manifest_metas,
                                    std::vector* merged_entries) const;
 
@@ -246,7 +272,10 @@ class FileStoreScan {
                                      std::vector* manifest_entries) const;
 
     Status ReadFileEntries(const std::vector& manifest_metas,
-                           std::vector* manifest_entries) const;
+                           std::vector* manifest_entries,
+                           bool apply_scan_filter) const;
+
+    bool MayContainBucket(const ManifestFileMeta& manifest, int32_t bucket) const;
 
     Result FilterManifestFileMeta(const ManifestFileMeta& manifest) const;
 
@@ -255,6 +284,8 @@ class FileStoreScan {
     Status ReadManifestFileMeta(const ManifestFileMeta& manifest,
                                 std::vector* entries) const;
 
+    Result FilterManifestEntry(const ManifestEntry& entry) const;
+
  protected:
     std::shared_ptr pool_;
     std::shared_ptr schema_manager_;
@@ -267,7 +298,6 @@ class FileStoreScan {
     CoreOptions core_options_;
 
  private:
-    mutable std::mutex lock_;
     bool only_read_real_buckets_ = false;
     std::shared_ptr snapshot_manager_;
     std::shared_ptr manifest_list_;
@@ -279,5 +309,6 @@ class FileStoreScan {
     std::function level_filter_;
     std::optional specified_snapshot_;
     std::shared_ptr metrics_;
+    std::string table_path_;
 };
 }  // namespace paimon
diff --git a/src/paimon/core/operation/file_store_scan_test.cpp b/src/paimon/core/operation/file_store_scan_test.cpp
index e35987ff..c4f0289e 100644
--- a/src/paimon/core/operation/file_store_scan_test.cpp
+++ b/src/paimon/core/operation/file_store_scan_test.cpp
@@ -18,13 +18,17 @@
 
 #include "paimon/core/operation/file_store_scan.h"
 
+#include 
+
 #include "arrow/type.h"
 #include "gtest/gtest.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_kind.h"
 #include "paimon/core/manifest/file_source.h"
+#include "paimon/core/manifest/snapshot_live_manifest_entries.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/data/timestamp.h"
+#include "paimon/memory/memory_segment.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -185,4 +189,88 @@ TEST_F(FileStoreScanTest, TestFilterManifestByRowRanges) {
     file_store_scan->WithRowRangeIndex(row_range_index);
     ASSERT_TRUE(file_store_scan->FilterManifestByRowRanges(manifest2));
 }
+
+TEST_F(FileStoreScanTest, TestSnapshotLiveManifestEntries) {
+    std::vector snapshot1;
+    ASSERT_OK_AND_ASSIGN(
+        auto file1,
+        DataFileMeta::ForAppend("file-1", /*file_size=*/10, /*row_count=*/1,
+                                SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+                                /*max_sequence_number=*/0, /*schema_id=*/0,
+                                /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+                                /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+                                /*write_cols=*/std::nullopt));
+    snapshot1.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0,
+                           /*total_buckets=*/1, file1);
+    SnapshotLiveManifestEntries entries(/*max_snapshots=*/2);
+    entries.Put(/*snapshot_id=*/1, std::move(snapshot1));
+    ASSERT_EQ(entries.Size(), 1);
+    auto hit = entries.LatestBeforeOrEqual(/*snapshot_id=*/1);
+    ASSERT_TRUE(hit);
+    ASSERT_EQ(hit->snapshot_id, 1);
+    ASSERT_EQ(hit->entries->size(), 1);
+    ASSERT_EQ((*hit->entries)[0].FileName(), "file-1");
+    auto latest_before_2 = entries.LatestBeforeOrEqual(/*snapshot_id=*/2);
+    ASSERT_TRUE(latest_before_2);
+    ASSERT_EQ(latest_before_2->snapshot_id, 1);
+
+    std::vector snapshot3;
+    ASSERT_OK_AND_ASSIGN(
+        auto file3,
+        DataFileMeta::ForAppend("file-3", /*file_size=*/10, /*row_count=*/1,
+                                SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+                                /*max_sequence_number=*/0, /*schema_id=*/0,
+                                /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+                                /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+                                /*write_cols=*/std::nullopt));
+    snapshot3.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0,
+                           /*total_buckets=*/1, file3);
+    entries.Put(/*snapshot_id=*/3, std::move(snapshot3));
+
+    auto latest_before_4 = entries.LatestBeforeOrEqual(/*snapshot_id=*/4);
+    ASSERT_TRUE(latest_before_4);
+    ASSERT_EQ(latest_before_4->snapshot_id, 3);
+
+    entries.Put(/*snapshot_id=*/5, {});
+    ASSERT_EQ(entries.Size(), 2);
+    ASSERT_FALSE(entries.LatestBeforeOrEqual(/*snapshot_id=*/1));
+    ASSERT_TRUE(entries.LatestBeforeOrEqual(/*snapshot_id=*/3));
+    ASSERT_TRUE(entries.LatestBeforeOrEqual(/*snapshot_id=*/5));
+}
+
+TEST_F(FileStoreScanTest, TestSnapshotLiveManifestEntriesSerialization) {
+    std::vector manifest_entries;
+    ASSERT_OK_AND_ASSIGN(
+        auto file1,
+        DataFileMeta::ForAppend("file-1", /*file_size=*/10, /*row_count=*/1,
+                                SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+                                /*max_sequence_number=*/0, /*schema_id=*/0,
+                                /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt,
+                                /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt,
+                                /*write_cols=*/std::nullopt));
+    manifest_entries.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0,
+                                  /*total_buckets=*/1, file1);
+    SnapshotLiveManifestEntries entries(/*max_snapshots=*/2);
+    entries.Put(/*snapshot_id=*/1, std::move(manifest_entries));
+    entries.Put(/*snapshot_id=*/3, {});
+
+    ASSERT_OK_AND_ASSIGN(auto bytes, entries.Serialize(GetDefaultPool()));
+    ASSERT_OK_AND_ASSIGN(auto deserialized,
+                         SnapshotLiveManifestEntries::Deserialize(
+                             MemorySegment::Wrap(bytes), /*max_snapshots=*/2, GetDefaultPool()));
+    ASSERT_EQ(deserialized.Size(), 2);
+    auto hit = deserialized.LatestBeforeOrEqual(/*snapshot_id=*/2);
+    ASSERT_TRUE(hit);
+    ASSERT_EQ(hit->snapshot_id, 1);
+    ASSERT_EQ(hit->entries->size(), 1);
+    ASSERT_EQ((*hit->entries)[0].FileName(), "file-1");
+    ASSERT_EQ(deserialized.LatestBeforeOrEqual(/*snapshot_id=*/4)->snapshot_id, 3);
+
+    ASSERT_OK_AND_ASSIGN(auto evicted_deserialized, SnapshotLiveManifestEntries::Deserialize(
+                                                        MemorySegment::Wrap(bytes),
+                                                        /*max_snapshots=*/1, GetDefaultPool()));
+    ASSERT_EQ(evicted_deserialized.Size(), 1);
+    ASSERT_FALSE(evicted_deserialized.LatestBeforeOrEqual(/*snapshot_id=*/1));
+    ASSERT_EQ(evicted_deserialized.LatestBeforeOrEqual(/*snapshot_id=*/3)->snapshot_id, 3);
+}
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp
index fb0f36b4..e8886b42 100644
--- a/src/paimon/core/operation/internal_read_context.cpp
+++ b/src/paimon/core/operation/internal_read_context.cpp
@@ -18,73 +18,207 @@
 
 #include "paimon/core/operation/internal_read_context.h"
 
+#include 
 #include 
 
+#include "arrow/api.h"
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
+#include "fmt/format.h"
+#include "paimon/common/data/variant/variant_access_utils.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/predicate/predicate_validator.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/schema/arrow_schema_validator.h"
+#include "paimon/core/utils/nested_projection_utils.h"
 #include "paimon/status.h"
 
-namespace arrow {
-class Schema;
-}  // namespace arrow
-
 namespace paimon {
+
+Result> InternalReadContext::AlignReadFieldWithTableFieldIds(
+    const std::shared_ptr& read_field,
+    const std::shared_ptr& table_field) {
+    static const std::vector kReadMetadataWhitelist = {DataField::MAP_SELECTED_KEYS};
+
+    if (VariantTypeUtils::IsVariantField(table_field) &&
+        VariantAccessUtils::IsVariantAccessType(read_field->type())) {
+        // A variant column may be read as a variant-access projection: a struct whose children
+        // each carry a `__VARIANT_METADATA` description. Keep the projection type (including
+        // the children's descriptions) on the aligned field.
+        return table_field->WithType(read_field->type());
+    }
+
+    if (read_field->type()->id() != table_field->type()->id()) {
+        return Status::Invalid(fmt::format(
+            "Read schema field '{}' type {} does not match table field type {}", read_field->name(),
+            read_field->type()->ToString(), table_field->type()->ToString()));
+    }
+
+    auto type_id = read_field->type()->id();
+    if (type_id == arrow::Type::STRUCT) {
+        auto read_struct = std::static_pointer_cast(read_field->type());
+        auto table_struct = std::static_pointer_cast(table_field->type());
+        arrow::FieldVector rebased_children;
+        rebased_children.reserve(read_struct->num_fields());
+        for (const auto& read_child : read_struct->fields()) {
+            auto table_child =
+                NestedProjectionUtils::FindFieldByName(table_struct->fields(), read_child->name());
+            if (!table_child) {
+                return Status::Invalid(fmt::format(
+                    "Read schema does not support schema evolution inside struct: nested field "
+                    "'{}' does not exist in table field '{}'",
+                    read_child->name(), read_field->name()));
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rebased_child,
+                                   AlignReadFieldWithTableFieldIds(read_child, table_child));
+            rebased_children.push_back(rebased_child);
+        }
+        auto rebased_type = arrow::struct_(rebased_children);
+        auto aligned_field = table_field->WithType(rebased_type);
+        return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field,
+                                                        kReadMetadataWhitelist);
+    }
+
+    if (type_id == arrow::Type::LIST) {
+        auto read_list = std::static_pointer_cast(read_field->type());
+        auto table_list = std::static_pointer_cast(table_field->type());
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr rebased_value_field,
+            AlignReadFieldWithTableFieldIds(read_list->value_field(), table_list->value_field()));
+        auto rebased_type = arrow::list(rebased_value_field);
+        auto aligned_field = table_field->WithType(rebased_type);
+        return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field,
+                                                        kReadMetadataWhitelist);
+    }
+
+    if (type_id == arrow::Type::MAP) {
+        auto read_map = std::static_pointer_cast(read_field->type());
+        auto table_map = std::static_pointer_cast(table_field->type());
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr rebased_key_field,
+            AlignReadFieldWithTableFieldIds(read_map->key_field(), table_map->key_field()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr rebased_item_field,
+            AlignReadFieldWithTableFieldIds(read_map->item_field(), table_map->item_field()));
+        auto rebased_type = arrow::map(rebased_key_field->type(), rebased_item_field);
+        auto aligned_field = table_field->WithType(rebased_type);
+        return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field,
+                                                        kReadMetadataWhitelist);
+    }
+
+    if (!read_field->type()->Equals(table_field->type())) {
+        return Status::Invalid(fmt::format(
+            "Read schema field '{}' type {} does not match table field type {}", read_field->name(),
+            read_field->type()->ToString(), table_field->type()->ToString()));
+    }
+
+    auto aligned_field = table_field->WithType(read_field->type());
+    return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field,
+                                                    kReadMetadataWhitelist);
+}
+
+std::optional InternalReadContext::TryResolveSpecialFieldById(
+    int32_t field_id, const CoreOptions& core_options) {
+    if (field_id == SpecialFields::ValueKind().Id()) {
+        return SpecialFields::ValueKind();
+    }
+    if (field_id == SpecialFields::RowId().Id()) {
+        if (core_options.RowTrackingEnabled()) {
+            return SpecialFields::RowId();
+        }
+        return std::nullopt;
+    }
+    if (field_id == SpecialFields::SequenceNumber().Id()) {
+        if (core_options.RowTrackingEnabled() || core_options.KeyValueSequenceNumberEnabled()) {
+            return SpecialFields::SequenceNumber();
+        }
+        return std::nullopt;
+    }
+    if (field_id == SpecialFields::IndexScore().Id()) {
+        if (core_options.DataEvolutionEnabled()) {
+            return SpecialFields::IndexScore();
+        }
+        return std::nullopt;
+    }
+    return std::nullopt;
+}
+
+std::optional InternalReadContext::TryResolveSpecialFieldByName(
+    const std::string& name, const CoreOptions& core_options) {
+    if (name == SpecialFields::ValueKind().Name()) {
+        return SpecialFields::ValueKind();
+    }
+    if (name == SpecialFields::RowId().Name()) {
+        if (core_options.RowTrackingEnabled()) {
+            return SpecialFields::RowId();
+        }
+        return std::nullopt;
+    }
+    if (name == SpecialFields::SequenceNumber().Name()) {
+        if (core_options.RowTrackingEnabled() || core_options.KeyValueSequenceNumberEnabled()) {
+            return SpecialFields::SequenceNumber();
+        }
+        return std::nullopt;
+    }
+    if (name == SpecialFields::IndexScore().Name()) {
+        if (core_options.DataEvolutionEnabled()) {
+            return SpecialFields::IndexScore();
+        }
+        return std::nullopt;
+    }
+    return std::nullopt;
+}
+
 Result> InternalReadContext::Create(
     const std::shared_ptr& context, const std::shared_ptr& table_schema,
     const std::map& options) {
     PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
                            CoreOptions::FromMap(options, context->GetSpecificFileSystem(),
                                                 context->GetFileSystemSchemeToIdentifierMap()));
+    core_options.WithCache(context->GetCache());
     // prepare read schema
+    // Priority: projected_arrow_schema > read_field_ids > read_field_names
+    const bool has_projected_read_schema = context->HasReadSchema();
     std::vector read_data_fields;
-    if (!context->GetReadFieldIds().empty()) {
-        read_data_fields.reserve(context->GetReadFieldIds().size());
-        for (const auto& field_id : context->GetReadFieldIds()) {
-            // if enable row tracking or data evolution, check special fields
-            if (core_options.RowTrackingEnabled() && field_id == SpecialFields::RowId().Id()) {
-                read_data_fields.push_back(SpecialFields::RowId());
-                continue;
-            }
-            if ((core_options.RowTrackingEnabled() ||
-                 core_options.KeyValueSequenceNumberEnabled()) &&
-                field_id == SpecialFields::SequenceNumber().Id()) {
-                read_data_fields.push_back(SpecialFields::SequenceNumber());
-                continue;
-            }
-            if (field_id == SpecialFields::ValueKind().Id()) {
-                read_data_fields.push_back(SpecialFields::ValueKind());
+    if (has_projected_read_schema) {
+        // Nested column pruning path: user provided a read C ArrowSchema
+        // where STRUCT types may contain only a subset of sub-fields.
+        // ImportSchema consumes the C schema — that's fine, it's one-shot usage.
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema,
+                                          arrow::ImportSchema(context->GetReadSchema()));
+        read_data_fields.reserve(read_schema->num_fields());
+        // Align special-field validation with read_field_ids/read_field_names branches.
+        for (const auto& read_field : read_schema->fields()) {
+            if (auto resolved_special_field =
+                    TryResolveSpecialFieldByName(read_field->name(), core_options)) {
+                read_data_fields.push_back(*resolved_special_field);
                 continue;
             }
-            if (core_options.DataEvolutionEnabled() &&
-                field_id == SpecialFields::IndexScore().Id()) {
-                read_data_fields.push_back(SpecialFields::IndexScore());
+            PAIMON_ASSIGN_OR_RAISE(DataField table_field,
+                                   table_schema->GetField(read_field->name()));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr aligned_field,
+                AlignReadFieldWithTableFieldIds(read_field, table_field.ArrowField()));
+            read_data_fields.emplace_back(table_field.Id(), aligned_field,
+                                          table_field.Description());
+        }
+    } else if (!context->GetReadFieldIds().empty()) {
+        read_data_fields.reserve(context->GetReadFieldIds().size());
+        for (const auto& field_id : context->GetReadFieldIds()) {
+            if (auto resolved_special_field = TryResolveSpecialFieldById(field_id, core_options)) {
+                read_data_fields.push_back(*resolved_special_field);
                 continue;
             }
             PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_id));
             read_data_fields.push_back(field);
         }
-    } else if (!context->GetReadSchema().empty()) {
-        read_data_fields.reserve(context->GetReadSchema().size());
-        for (const auto& name : context->GetReadSchema()) {
-            // if enable row tracking or data evolution, check special fields
-            if (core_options.RowTrackingEnabled() && name == SpecialFields::RowId().Name()) {
-                read_data_fields.push_back(SpecialFields::RowId());
-                continue;
-            }
-            if ((core_options.RowTrackingEnabled() ||
-                 core_options.KeyValueSequenceNumberEnabled()) &&
-                name == SpecialFields::SequenceNumber().Name()) {
-                read_data_fields.push_back(SpecialFields::SequenceNumber());
-                continue;
-            }
-            if (name == SpecialFields::ValueKind().Name()) {
-                read_data_fields.push_back(SpecialFields::ValueKind());
-                continue;
-            }
-            if (core_options.DataEvolutionEnabled() && name == SpecialFields::IndexScore().Name()) {
-                read_data_fields.push_back(SpecialFields::IndexScore());
+    } else if (!context->GetReadFieldNames().empty()) {
+        read_data_fields.reserve(context->GetReadFieldNames().size());
+        for (const auto& name : context->GetReadFieldNames()) {
+            if (auto resolved_special_field = TryResolveSpecialFieldByName(name, core_options)) {
+                read_data_fields.push_back(*resolved_special_field);
                 continue;
             }
             PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(name));
@@ -95,8 +229,14 @@ Result> InternalReadContext::Create(
         read_data_fields = table_schema->Fields();
     }
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_data_fields);
-    // validate read schema to avoid redundant fields
-    PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema));
+    // validate read schema to avoid redundant fields.
+    // For projected read schema, nested sub-fields may be user-requested fields
+    // that do not exist in table schema, so they may not have paimon field IDs.
+    if (has_projected_read_schema) {
+        PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchema(*read_schema));
+    } else {
+        PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema));
+    }
     // validate predicate
     if (context->GetPredicate()) {
         PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
@@ -118,4 +258,13 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea
       read_schema_(read_schema),
       options_(options) {}
 
+Result> InternalReadContext::CreateWithSchema(
+    const std::shared_ptr& original,
+    const std::shared_ptr& new_read_schema) {
+    // Create a new InternalReadContext sharing all properties except read_schema.
+    // The new read_schema is the minimal column set for COUNT(*).
+    return std::shared_ptr(new InternalReadContext(
+        original->read_context_, original->table_schema_, new_read_schema, original->options_));
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h
index c1990264..e0474b76 100644
--- a/src/paimon/core/operation/internal_read_context.h
+++ b/src/paimon/core/operation/internal_read_context.h
@@ -100,12 +100,28 @@ class InternalReadContext {
         return read_context_->GetCacheConfig();
     }
 
+    /// Create a new InternalReadContext with a different read schema.
+    /// Useful for creating a context with a minimal column set for specialized reads.
+    /// All other settings (predicate, options, table_schema, etc.) are inherited
+    /// from the original context.
+    static Result> CreateWithSchema(
+        const std::shared_ptr& original,
+        const std::shared_ptr& new_read_schema);
+
  private:
     InternalReadContext(const std::shared_ptr& read_context,
                         const std::shared_ptr& table_schema,
                         const std::shared_ptr& read_schema,
                         const CoreOptions& options);
 
+    static std::optional TryResolveSpecialFieldById(int32_t field_id,
+                                                               const CoreOptions& core_options);
+    static std::optional TryResolveSpecialFieldByName(const std::string& name,
+                                                                 const CoreOptions& core_options);
+    static Result> AlignReadFieldWithTableFieldIds(
+        const std::shared_ptr& read_field,
+        const std::shared_ptr& table_field);
+
     std::shared_ptr read_context_;
     std::shared_ptr table_schema_;
     std::shared_ptr read_schema_;
diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp
index 4d1467b2..30ba77b7 100644
--- a/src/paimon/core/operation/internal_read_context_test.cpp
+++ b/src/paimon/core/operation/internal_read_context_test.cpp
@@ -52,7 +52,7 @@ TEST(InternalReadContext, TestReadWithUnspecifiedSchema) {
 TEST(InternalReadContext, TestReadWithSpecifiedSchema) {
     std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f3", "f0"});
+    context_builder.SetReadFieldNames({"f3", "f0"});
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
     SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
     ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -86,7 +86,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) {
     ReadContextBuilder context_builder(path);
     // read schema is specified, read fields in schema
     // will use field ids instead of field names.
-    context_builder.SetReadSchema({"f0"});
+    context_builder.SetReadFieldNames({"f0"});
     context_builder.SetReadFieldIds({3, 0});
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
     SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
@@ -105,7 +105,8 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) {
         // test simple
         std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"});
+        context_builder.SetReadFieldNames(
+            {"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"});
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
         ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -126,7 +127,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) {
         // test invalid case: disable row tracking while read row tracking fields
         std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"});
+        context_builder.SetReadFieldNames({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"});
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
         ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -138,7 +139,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) {
         // test invalid case: disable data evolution while read score fields
         std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f3", "f0", "_INDEX_SCORE"});
+        context_builder.SetReadFieldNames({"f3", "f0", "_INDEX_SCORE"});
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
         ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -151,7 +152,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) {
 TEST(InternalReadContext, TestReadWithValueKindField) {
     std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f3", "_VALUE_KIND", "f0"});
+    context_builder.SetReadFieldNames({"f3", "_VALUE_KIND", "f0"});
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
     SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
     ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -193,4 +194,113 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) {
     }
 }
 
+TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) {
+    std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+
+    std::vector projected_fields = {
+        DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(),
+        SpecialFields::SequenceNumber(), SpecialFields::IndexScore()};
+    auto schema_manager = SchemaManager(std::make_shared(), path);
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
+
+    // Without options, special fields should be rejected in projected-schema path too.
+    {
+        auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields);
+        auto c_schema = std::make_unique();
+        ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+        ReadContextBuilder context_builder(path);
+        context_builder.SetReadSchema(std::move(c_schema));
+        ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish());
+        std::shared_ptr read_context = std::move(unique_read_context);
+        ASSERT_NOK_WITH_MSG(
+            InternalReadContext::Create(read_context, table_schema, table_schema->Options()),
+            "not exist in table schema");
+    }
+
+    // With options enabled, projected-schema path should accept these special fields.
+    auto enabled_options = table_schema->Options();
+    enabled_options[Options::ROW_TRACKING_ENABLED] = "true";
+    enabled_options[Options::DATA_EVOLUTION_ENABLED] = "true";
+
+    {
+        auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields);
+        auto c_schema = std::make_unique();
+        ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+        ReadContextBuilder context_builder(path);
+        context_builder.SetReadSchema(std::move(c_schema));
+        ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish());
+        std::shared_ptr read_context = std::move(unique_read_context);
+        ASSERT_OK_AND_ASSIGN(
+            auto internal_context,
+            InternalReadContext::Create(read_context, table_schema, enabled_options));
+        auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields);
+        ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema));
+    }
+}
+
+TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) {
+    std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+
+    auto projected_schema =
+        arrow::schema({arrow::field("f3", arrow::float64()), arrow::field("f0", arrow::utf8())});
+    auto c_schema = std::make_unique();
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadSchema(std::move(c_schema));
+    ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish());
+    std::shared_ptr read_context = std::move(unique_read_context);
+
+    SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
+
+    ASSERT_OK_AND_ASSIGN(
+        auto internal_context,
+        InternalReadContext::Create(read_context, table_schema, table_schema->Options()));
+
+    std::vector expected_fields = {
+        DataField(3, arrow::field("f3", arrow::float64())),
+        DataField(0, arrow::field("f0", arrow::utf8())),
+    };
+    auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(expected_fields);
+    ASSERT_TRUE(
+        internal_context->GetReadSchema()->Equals(expected_schema, /*check_metadata=*/true));
+}
+
+TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) {
+    std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
+
+    auto read_field =
+        arrow::field("f0", arrow::utf8())
+            ->WithMetadata(arrow::KeyValueMetadata::Make(
+                {DataField::MAP_SELECTED_KEYS, "custom.key"}, {"k1,k2", "should_not_propagate"}));
+    auto projected_schema = arrow::schema({read_field});
+    auto c_schema = std::make_unique();
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+
+    ReadContextBuilder context_builder(path);
+    context_builder.SetReadSchema(std::move(c_schema));
+    ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish());
+    std::shared_ptr read_context = std::move(unique_read_context);
+
+    SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
+    ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
+
+    ASSERT_OK_AND_ASSIGN(
+        auto internal_context,
+        InternalReadContext::Create(read_context, table_schema, table_schema->Options()));
+
+    auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0");
+    ASSERT_TRUE(aligned_field);
+    ASSERT_TRUE(aligned_field->HasMetadata());
+    ASSERT_TRUE(aligned_field->metadata());
+
+    auto selected_keys_result = aligned_field->metadata()->Get(DataField::MAP_SELECTED_KEYS);
+    ASSERT_TRUE(selected_keys_result.ok());
+    ASSERT_EQ(selected_keys_result.ValueUnsafe(), "k1,k2");
+
+    auto custom_metadata_result = aligned_field->metadata()->Get("custom.key");
+    ASSERT_FALSE(custom_metadata_result.ok());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp
index 80f6b807..54197f87 100644
--- a/src/paimon/core/operation/key_value_file_store_scan.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan.cpp
@@ -35,6 +35,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/options/merge_engine.h"
+#include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/stats/simple_stats_evolution.h"
@@ -70,6 +71,7 @@ Result> KeyValueFileStoreScan::Create(
         scan->SplitAndSetFilter(table_schema->PartitionKeys(), arrow_schema, scan_filters));
     PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk, table_schema->TrimmedPrimaryKeys());
     PAIMON_RETURN_NOT_OK(scan->SplitAndSetKeyValueFilter(trimmed_pk));
+
     return scan;
 }
 
@@ -190,14 +192,25 @@ Result KeyValueFileStoreScan::FilterByValueFilter(const ManifestEntry& ent
 
     const auto& meta = entry.File();
 
-    // Primary key table currently does not support schema evolution for value filtering.
-    // Here we only handle `value_stats_cols` (dense stats) projection.
-    if (meta->schema_id != table_schema_->Id()) {
-        return Status::NotImplemented(
-            "Primary key table does not support schema evolution in FilterByValueFilter");
+    std::shared_ptr data_schema = table_schema_;
+    std::shared_ptr trimmed_predicates = value_filter_;
+    int64_t data_schema_id = meta->schema_id;
+    if (data_schema_id != table_schema_->Id()) {
+        PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(data_schema_id));
     }
 
-    auto evolution = evolutions_->GetOrCreate(table_schema_);
+    auto evolution = evolutions_->GetOrCreate(data_schema);
+    if (data_schema_id != table_schema_->Id()) {
+        PAIMON_ASSIGN_OR_RAISE(trimmed_predicates,
+                               ReconstructPredicateWithNonCastedFields(value_filter_, evolution));
+    }
+    if (!trimmed_predicates) {
+        return true;
+    }
+    auto predicate_filter = std::dynamic_pointer_cast(trimmed_predicates);
+    if (!predicate_filter) {
+        return Status::Invalid("invalid value predicate, cannot cast to PredicateFilter");
+    }
 
     PAIMON_ASSIGN_OR_RAISE(
         SimpleStatsEvolution::EvolutionStats new_stats,
@@ -206,8 +219,8 @@ Result KeyValueFileStoreScan::FilterByValueFilter(const ManifestEntry& ent
     try {
         PAIMON_ASSIGN_OR_RAISE(
             bool predicate_result,
-            value_filter_->Test(schema_, meta->row_count, *(new_stats.min_values),
-                                *(new_stats.max_values), *(new_stats.null_counts)));
+            predicate_filter->Test(schema_, meta->row_count, *(new_stats.min_values),
+                                   *(new_stats.max_values), *(new_stats.null_counts)));
         return predicate_result;
     } catch (const std::exception& e) {
         return Status::Invalid(fmt::format("FilterByValueFilter failed for file {}, with {} error",
diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
index 04314039..b221715f 100644
--- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp
@@ -97,7 +97,7 @@ class KeyValueFileStoreScanTest : public testing::Test {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr manifest_list,
             ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(),
-                                 path_factory, pool_));
+                                 path_factory, core_options.GetCache(), pool_));
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr partition_schema,
             FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys()));
@@ -439,4 +439,135 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) {
     ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(entry_keep));
     ASSERT_TRUE(keep);
 }
+
+TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) {
+    std::string table_path =
+        paimon::test::GetDataDir() + "orc/pk_table_with_alter_table.db/pk_table_with_alter_table";
+    std::vector> partition_filters = {};
+
+    // In schema-1, `c` is renamed from schema-0 field `b` and moves to index 3.
+    auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"c",
+                                                      FieldType::INT, Literal(30));
+    auto scan_filter = std::make_shared(/*predicate=*/greater_than,
+                                                    /*partition_filters=*/partition_filters,
+                                                    /*bucket_filter=*/0);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr scan,
+                         CreateFileStoreScan(table_path, scan_filter,
+                                             /*table_schema_id=*/1, /*snapshot_id=*/6));
+    scan->EnableValueFilter();
+
+    auto pool = GetDefaultPool();
+    // Build schema-0 dense stats for field `b`; after evolution they are tested against
+    // schema-1 field `c`.
+    SimpleStats value_stats = BinaryRowGenerator::GenerateStats(
+        /*min=*/{10}, /*max=*/{20}, /*null=*/{0}, pool.get());
+    std::vector value_stats_cols = {"b"};
+    ManifestEntry entry(
+        /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0,
+        /*total_buckets=*/1,
+        std::make_shared(
+            /*file_name=*/"schema0_name", /*file_size=*/1024, /*row_count=*/10,
+            /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
+            /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/value_stats,
+            /*min_sequence_number=*/0,
+            /*max_sequence_number=*/10,
+            /*schema_id=*/0,
+            /*level=*/1,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr,
+            /*file_source=*/FileSource::Append(),
+            /*value_stats_cols=*/value_stats_cols,
+            /*external_path=*/std::nullopt,
+            /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt));
+
+    SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats(
+        /*min=*/{40}, /*max=*/{50}, /*null=*/{0}, pool.get());
+    ManifestEntry entry_keep(
+        /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0,
+        /*total_buckets=*/1,
+        std::make_shared(
+            /*file_name=*/"schema0_name_keep", /*file_size=*/1024, /*row_count=*/10,
+            /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
+            /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/value_stats_keep,
+            /*min_sequence_number=*/0,
+            /*max_sequence_number=*/10,
+            /*schema_id=*/0,
+            /*level=*/1,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr,
+            /*file_source=*/FileSource::Append(),
+            /*value_stats_cols=*/value_stats_cols,
+            /*external_path=*/std::nullopt,
+            /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry));
+    ASSERT_FALSE(keep);
+
+    ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(entry_keep));
+    ASSERT_TRUE(keep);
+}
+
+TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithNewFieldUsesNullStats) {
+    std::string table_path =
+        paimon::test::GetDataDir() + "orc/pk_table_with_alter_table.db/pk_table_with_alter_table";
+    std::vector> partition_filters = {};
+
+    // `e` only exists in schema-1. Schema-0 files expose it to stats filtering as all NULL.
+    auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/7, /*field_name=*/"e",
+                                                      FieldType::INT, Literal(30));
+    auto scan_filter = std::make_shared(/*predicate=*/greater_than,
+                                                    /*partition_filters=*/partition_filters,
+                                                    /*bucket_filter=*/0);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr scan,
+                         CreateFileStoreScan(table_path, scan_filter,
+                                             /*table_schema_id=*/1, /*snapshot_id=*/6));
+    scan->EnableValueFilter();
+
+    auto pool = GetDefaultPool();
+    SimpleStats value_stats = BinaryRowGenerator::GenerateStats(
+        /*min=*/{10}, /*max=*/{20}, /*null=*/{0}, pool.get());
+    std::vector value_stats_cols = {"b"};
+    ManifestEntry old_schema_entry(
+        /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0,
+        /*total_buckets=*/1,
+        std::make_shared(
+            /*file_name=*/"schema0_missing_e", /*file_size=*/1024, /*row_count=*/10,
+            /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(),
+            /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/value_stats,
+            /*min_sequence_number=*/0,
+            /*max_sequence_number=*/10,
+            /*schema_id=*/0,
+            /*level=*/1,
+            /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0),
+            /*delete_row_count=*/std::nullopt,
+            /*embedded_index=*/nullptr,
+            /*file_source=*/FileSource::Append(),
+            /*value_stats_cols=*/value_stats_cols,
+            /*external_path=*/std::nullopt,
+            /*first_row_id=*/std::nullopt,
+            /*write_cols=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(old_schema_entry));
+    ASSERT_FALSE(keep);
+
+    auto is_null = PredicateBuilder::IsNull(/*field_index=*/7, /*field_name=*/"e", FieldType::INT);
+    scan_filter = std::make_shared(/*predicate=*/is_null,
+                                               /*partition_filters=*/partition_filters,
+                                               /*bucket_filter=*/0);
+    ASSERT_OK_AND_ASSIGN(scan, CreateFileStoreScan(table_path, scan_filter,
+                                                   /*table_schema_id=*/1, /*snapshot_id=*/6));
+    scan->EnableValueFilter();
+    ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(old_schema_entry));
+    ASSERT_TRUE(keep);
+}
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 0f511993..08c5ea0c 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -81,7 +81,8 @@ Result> KeyValueFileStoreWrite::CreateFileStoreSc
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_list,
         ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
-                             options_.GetManifestCompression(), file_store_path_factory_, pool_));
+                             options_.GetManifestCompression(), file_store_path_factory_,
+                             options_.GetCache(), pool_));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_file,
         ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(),
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index be4c7153..60f2d95a 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -20,6 +20,7 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -30,13 +31,25 @@
 #include "arrow/c/abi.h"
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
+#include "arrow/ipc/json_simple.h"
 #include "arrow/type.h"
 #include "gtest/gtest.h"
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/map_shredding_defs.h"
+#include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/file_store_commit.h"
 #include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/reader/file_batch_reader.h"
 #include "paimon/record_batch.h"
 #include "paimon/status.h"
 #include "paimon/testing/utils/test_helper.h"
@@ -99,6 +112,84 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
         }
         return write_status;
     }
+
+    void CreateTable(const std::string& warehouse, const std::shared_ptr& schema,
+                     const std::map& options) const {
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
+        ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false));
+        ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema,
+                                       /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options,
+                                       /*ignore_if_exists=*/false));
+    }
+
+    std::unique_ptr MakeBatch(const std::shared_ptr& schema,
+                                           const std::string& json) const {
+        auto struct_type = arrow::struct_(schema->fields());
+        auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie();
+        ::ArrowArray arrow_array;
+        EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok());
+        RecordBatchBuilder batch_builder(&arrow_array);
+        return batch_builder.SetBucket(0).Finish().value();
+    }
+
+    std::vector> WriteAndPrepare(
+        const std::string& table_path, const std::shared_ptr& schema,
+        const std::map& options, const std::string& json,
+        int64_t commit_identifier) const {
+        WriteContextBuilder builder(table_path, "test");
+        builder.SetOptions(options);
+        EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+        EXPECT_OK_AND_ASSIGN(auto file_store_write,
+                             FileStoreWrite::Create(std::move(write_context)));
+        EXPECT_OK(file_store_write->Write(MakeBatch(schema, json)));
+        EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                                   /*wait_compaction=*/false, commit_identifier));
+        EXPECT_OK(file_store_write->Close());
+        return commit_msgs;
+    }
+
+    std::shared_ptr OnlyNewFile(
+        const std::vector>& commit_msgs) const {
+        EXPECT_EQ(1, commit_msgs.size());
+        auto msg = std::dynamic_pointer_cast(commit_msgs[0]);
+        EXPECT_NE(nullptr, msg);
+        EXPECT_EQ(1, msg->GetNewFilesIncrement().NewFiles().size());
+        return msg->GetNewFilesIncrement().NewFiles()[0];
+    }
+
+    void Commit(const std::string& table_path, const std::map& options,
+                const std::vector>& commit_msgs) const {
+        CommitContextBuilder builder(table_path, "test");
+        builder.SetOptions(options);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish());
+        ASSERT_OK_AND_ASSIGN(auto file_store_commit,
+                             FileStoreCommit::Create(std::move(commit_context)));
+        ASSERT_OK(file_store_commit->Commit(commit_msgs));
+    }
+
+    std::shared_ptr ReadDataFileSchema(
+        const std::string& table_path, const std::shared_ptr& file,
+        const std::map& options) const {
+        std::string file_path =
+            PathUtil::JoinPath(PathUtil::JoinPath(table_path, "bucket-0"), file->file_name);
+        auto fs = std::make_shared();
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path));
+        EXPECT_OK_AND_ASSIGN(auto format_str, file->FileFormat());
+        EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format_str, options));
+        EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(10));
+        EXPECT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream));
+        EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema());
+        return arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
+    }
+
+    MapSharedShreddingFieldMeta ShreddingMeta(const std::shared_ptr& file_schema,
+                                              int32_t field_index) const {
+        auto metadata = file_schema->field(field_index)->metadata();
+        EXPECT_NE(nullptr, metadata);
+        return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
+    }
 };
 
 TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) {
@@ -221,6 +312,61 @@ TEST_F(KeyValueFileStoreWriteTest,
     ASSERT_EQ(commit_messages.size(), 1);
 }
 
+TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFiles) {
+    std::map options = {
+        {"file.format", "parquet"},
+        {"target-file-row-num", "2"},
+        {"write.batch-size", "2"},
+        {"fields.tags.map.storage-layout", "shared-shredding"},
+        {"fields.tags.map.shared-shredding.max-columns", "10"},
+        {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
+        {"write-only", "true"},
+        {"bucket", "1"},
+    };
+    auto logical_schema = arrow::schema({
+        arrow::field("id", arrow::int32(), /*nullable=*/false),
+        arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(logical_schema);
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    CreateTable(dir->Str(), logical_schema, options);
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    auto commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+        [1, [["a", 1], ["b", 2]]],
+        [2, [["c", 3], ["d", 4], ["e", 5]]],
+        [3, [["f", 6]]],
+        [4, [["g", 7], ["h", 8]]]
+    ])",
+                                       /*commit_identifier=*/0);
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
+    auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_first_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, first_meta.num_columns);
+    ASSERT_EQ(3, first_meta.max_row_width);
+
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
+    auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_second_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 3}}));
+    ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false));
+    ASSERT_EQ(3, second_meta.num_columns);
+    ASSERT_EQ(2, second_meta.max_row_width);
+}
+
 TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) {
     auto fields = {arrow::field("f0", arrow::utf8(), /*nullable=*/false)};
     arrow::Schema typed_schema(fields);
diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp
index 150723f6..1ff1ffd7 100644
--- a/src/paimon/core/operation/manifest_file_merger_test.cpp
+++ b/src/paimon/core/operation/manifest_file_merger_test.cpp
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -111,9 +112,22 @@ class ManifestFileMergerTest : public testing::Test {
         return manifest_file_metas[0];
     }
 
+    std::set ListManifestFiles() const {
+        std::vector> file_statuses;
+        EXPECT_OK(file_system_->ListFileStatus(
+            FileStorePathFactory::ManifestPath(path_factory_->RootPath()), &file_statuses));
+        std::set files;
+        for (const auto& status : file_statuses) {
+            if (!status->IsDir()) {
+                files.insert(status->GetPath());
+            }
+        }
+        return files;
+    }
+
  private:
     void CreateManifestFile(const std::string& path_str) {
-        auto file_system = std::make_shared();
+        file_system_ = std::make_shared();
         ASSERT_OK_AND_ASSIGN(
             std::shared_ptr file_format,
             FileFormatFactory::Get("parquet", std::map()));
@@ -131,10 +145,11 @@ class ManifestFileMergerTest : public testing::Test {
                 options.GetFileFormat()->Identifier(), options.DataFilePrefix(),
                 options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path,
                 options.IndexFileInDataFileDir(), pool_));
+        path_factory_ = path_factory;
         ASSERT_OK_AND_ASSIGN(std::shared_ptr partition_schema,
                              FieldMapping::GetPartitionSchema(schema, {"f0"}));
         ASSERT_OK_AND_ASSIGN(manifest_file_,
-                             ManifestFile::Create(file_system, file_format, "zstd", path_factory,
+                             ManifestFile::Create(file_system_, file_format, "zstd", path_factory,
                                                   /*target_file_size=*/1024 * 1024, pool_, options,
                                                   partition_schema));
     }
@@ -168,6 +183,8 @@ class ManifestFileMergerTest : public testing::Test {
     std::string test_root_;
     std::shared_ptr pool_;
     std::shared_ptr manifest_file_;
+    std::shared_ptr file_system_;
+    std::shared_ptr path_factory_;
     std::shared_ptr partition_type_;
 };
 
@@ -372,4 +389,24 @@ TEST_F(ManifestFileMergerTest, TestTriggerFullCompaction) {
     ContainSameEntryFile(merged.value(), entry_file_expected);
 }
 
+TEST_F(ManifestFileMergerTest, TestDeleteNewManifestFilesWhenMinorCompactionFails) {
+    std::vector input;
+    for (int32_t i = 0; i < 4; i++) {
+        input.push_back(MakeManifest({MakeEntry(FileKind::Add(), std::to_string(i))}));
+    }
+
+    // The first pair is merged successfully. Removing a source file from the second pair makes
+    // the following merge fail after a new manifest file has already been created.
+    std::string missing_manifest_path = path_factory_->ToManifestFilePath(input[2].FileName());
+    ASSERT_OK(file_system_->Delete(missing_manifest_path));
+    std::set files_before_merge = ListManifestFiles();
+
+    ASSERT_NOK_WITH_MSG(ManifestFileMerger::Merge(
+                            input, /*manifest_target_file_size=*/5000, /*merge_min_count=*/2,
+                            /*full_compaction_file_size=*/MAX_LONG_VALUE, manifest_file_.get()),
+                        "not exists");
+
+    ASSERT_EQ(files_before_merge, ListManifestFiles());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index ca6c1d57..2f2b46cf 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -235,8 +235,10 @@ Result> MergeFileSplitRead::ApplyIndexAndDvRead
 Result> MergeFileSplitRead::CreateMergeReader(
     const std::shared_ptr& data_split,
     const std::shared_ptr& data_file_path_factory) {
-    auto dv_factory = DeletionVector::CreateFactory(options_.GetFileSystem(),
-                                                    CreateDeletionFileMap(*data_split), pool_);
+    auto dv_factory = DeletionVector::CreateFactory(
+        options_.GetFileSystem(),
+        DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()),
+        pool_);
 
     std::vector> sections =
         IntervalPartition(data_split->DataFiles(), key_comparator_).Partition();
@@ -257,19 +259,23 @@ Result> MergeFileSplitRead::CreateMergeReader(
 Result> MergeFileSplitRead::CreateNoMergeReader(
     const std::shared_ptr& data_split, bool only_filter_key,
     const std::shared_ptr& data_file_path_factory) const {
-    auto dv_factory = DeletionVector::CreateFactory(options_.GetFileSystem(),
-                                                    CreateDeletionFileMap(*data_split), pool_);
+    auto dv_factory = DeletionVector::CreateFactory(
+        options_.GetFileSystem(),
+        DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()),
+        pool_);
 
     // create read schema without extra fields (e.g., completed key, sequence fields)
-    auto row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
-
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema,
-                                      raw_read_schema_->AddField(0, row_kind_field));
+    std::shared_ptr read_schema = raw_read_schema_;
+    if (read_schema->GetFieldIndex(SpecialFields::ValueKind().Name()) < 0) {
+        auto row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(read_schema, read_schema->AddField(0, row_kind_field));
+    }
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema,
                              only_filter_key ? predicate_for_keys_ : context_->GetPredicate(),
-                             dv_factory, /*row_ranges=*/{}, data_file_path_factory));
+                             dv_factory, /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     auto raw_readers =
         ObjectUtils::MoveVector>(std::move(raw_file_readers));
@@ -493,7 +499,8 @@ Result> MergeFileSplitRead::CreateReaderFo
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory,
-                             /*row_ranges=*/{}, data_file_path_factory));
+                             /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     assert(data_files.size() == raw_file_readers.size());
     // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects
diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h
index f00883b0..895bb573 100644
--- a/src/paimon/core/operation/merge_file_split_read.h
+++ b/src/paimon/core/operation/merge_file_split_read.h
@@ -73,8 +73,9 @@ class MergeFunctionWrapper;
 /// ->ConcatBatchReader across no overlapped
 /// files->KeyValueProjectionReader/AsyncKeyValueProjectionReader
 /// ->DropDeleteReader->SortMergeReader->ConcatKeyValueRecordReader->KeyValueDataFileRecordReader
-/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(DelegatingPrefetchReader)
-/// ->(PrefetchFileBatchReader)->FormatReader
+/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ShreddingFileReader)
+/// ->(MapSharedShreddingFileReader)
+/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
 class MergeFileSplitRead : public AbstractSplitRead {
  public:
     static Result> Create(
@@ -107,6 +108,11 @@ class MergeFileSplitRead : public AbstractSplitRead {
         return path_factory_;
     }
 
+    /// Get the key comparator (needed by count readers for IntervalPartition).
+    std::shared_ptr GetKeyComparator() const {
+        return key_comparator_;
+    }
+
     std::shared_ptr GetValueSchema() const {
         return value_schema_;
     }
diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp
index 70383b22..911cf796 100644
--- a/src/paimon/core/operation/merge_file_split_read_test.cpp
+++ b/src/paimon/core/operation/merge_file_split_read_test.cpp
@@ -363,7 +363,7 @@ class MergeFileSplitReadTest : public ::testing::Test,
  private:
     std::shared_ptr pool_ = GetDefaultPool();
     std::shared_ptr fs_ = std::make_shared();
-    std::shared_ptr executor_ = CreateDefaultExecutor(/*thread_count=*/4);
+    std::shared_ptr executor_ = CreateDefaultExecutor();
 };
 
 // test GenerateKeyValueReadSchema with user define fields
@@ -613,7 +613,7 @@ TEST_P(MergeFileSplitReadTest, TestSimple) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
@@ -679,7 +679,7 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"},
@@ -753,7 +753,7 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"});
+    context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"});
     context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}});
     AddOptions(&context_builder);
     ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish());
@@ -794,7 +794,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "s0", "v0", "v1"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
@@ -859,7 +859,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"});
+    context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
@@ -908,7 +908,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"v2", "p1", "k0", "p0", "s0", "v0"});
+    context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
@@ -956,7 +956,7 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "aggregation"},
                                 {"fields.v1.aggregate-function", "bool_and"},
@@ -1003,7 +1003,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "v0"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "partial-update"},
                                 {"fields.v1.sequence-group", "v0"},
@@ -1051,7 +1051,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"});
+    context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"});
     context_builder.SetOptions(
         {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}});
     AddOptions(&context_builder);
@@ -1091,7 +1091,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"});
+    context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"});
     context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"},
                                 {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}});
     AddOptions(&context_builder);
@@ -1131,7 +1131,7 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"});
+    context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"});
     context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"},
                                 {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}});
     AddOptions(&context_builder);
@@ -1158,7 +1158,7 @@ TEST_P(MergeFileSplitReadTest, TestIOException) {
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
 
-    context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"});
+    context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"});
     context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {Options::IGNORE_DELETE, "true"}});
@@ -1212,7 +1212,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) {
                                               DataField(1, arrow::field("f1", arrow::int32()))};
     auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields);
     ASSERT_TRUE(read_schema);
-    context_builder.SetReadSchema({"f3", "f2", "f0", "f1"});
+    context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"});
     context_builder.SetOptions({{Options::FILE_FORMAT, "orc"},
                                 {Options::MERGE_ENGINE, "deduplicate"},
                                 {"orc.read.enable-metrics", "true"}});
diff --git a/src/paimon/core/operation/metrics/commit_metrics.cpp b/src/paimon/core/operation/metrics/commit_metrics.cpp
new file mode 100644
index 00000000..76880932
--- /dev/null
+++ b/src/paimon/core/operation/metrics/commit_metrics.cpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/metrics/commit_metrics.h"
+
+#include "paimon/core/operation/metrics/commit_stats.h"
+#include "paimon/metrics.h"
+
+namespace paimon {
+
+void CommitMetrics::ReportCommit(const std::shared_ptr& metrics,
+                                 const CommitStats& commit_stats) {
+    metrics->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, commit_stats.GetDuration());
+    metrics->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, commit_stats.GetAttempts());
+    metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, commit_stats.GetTableFilesAdded());
+    metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED,
+                        commit_stats.GetTableFilesDeleted());
+    metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED,
+                        commit_stats.GetTableFilesAppended());
+    metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED,
+                        commit_stats.GetTableFilesCompacted());
+    metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED,
+                        commit_stats.GetChangelogFilesAppended());
+    metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED,
+                        commit_stats.GetChangelogFilesCompacted());
+    metrics->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS,
+                        commit_stats.GetGeneratedSnapshots());
+    metrics->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED,
+                        commit_stats.GetDeltaRecordsAppended());
+    metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED,
+                        commit_stats.GetChangelogRecordsAppended());
+    metrics->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED,
+                        commit_stats.GetDeltaRecordsCompacted());
+    metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED,
+                        commit_stats.GetChangelogRecordsCompacted());
+    metrics->SetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN,
+                        commit_stats.GetNumPartitionsWritten());
+    metrics->SetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN, commit_stats.GetNumBucketsWritten());
+    metrics->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE,
+                        commit_stats.GetCompactionInputFileSize());
+    metrics->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE,
+                        commit_stats.GetCompactionOutputFileSize());
+    metrics->SetCounter(CommitMetrics::LAST_COMMITTED_SNAPSHOT_ID,
+                        commit_stats.GetLastCommittedSnapshotId());
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/metrics/commit_metrics.h b/src/paimon/core/operation/metrics/commit_metrics.h
index cb26a121..ce9c25ce 100644
--- a/src/paimon/core/operation/metrics/commit_metrics.h
+++ b/src/paimon/core/operation/metrics/commit_metrics.h
@@ -19,8 +19,13 @@
 
 #pragma once
 
+#include 
+
 namespace paimon {
 
+class CommitStats;
+class Metrics;
+
 /// Metrics to measure a commit.
 class CommitMetrics {
  public:
@@ -43,6 +48,10 @@ class CommitMetrics {
     static constexpr char LAST_BUCKETS_WRITTEN[] = "lastBucketsWritten";
     static constexpr char LAST_COMPACTION_INPUT_FILE_SIZE[] = "lastCompactionInputFileSize";
     static constexpr char LAST_COMPACTION_OUTPUT_FILE_SIZE[] = "lastCompactionOutputFileSize";
+    static constexpr char LAST_COMMITTED_SNAPSHOT_ID[] = "lastCommittedSnapshotId";
+
+    static void ReportCommit(const std::shared_ptr& metrics,
+                             const CommitStats& commit_stats);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/metrics/commit_metrics_test.cpp b/src/paimon/core/operation/metrics/commit_metrics_test.cpp
deleted file mode 100644
index 787a01dd..00000000
--- a/src/paimon/core/operation/metrics/commit_metrics_test.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/operation/metrics/commit_metrics.h"
-
-#include 
-#include 
-
-#include "gtest/gtest.h"
-#include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-
-TEST(CommitMetricsTest, TestSimple) {
-    auto commit_metrics = std::make_shared();
-    commit_metrics->SetCounter("some_metric", 100);
-    commit_metrics->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, 30);
-    ASSERT_OK_AND_ASSIGN(uint64_t counter,
-                         commit_metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
-    ASSERT_EQ(30, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric"));
-    ASSERT_EQ(100, counter);
-    auto other = std::make_shared();
-    other->SetCounter("some_metric_2", 200);
-    other->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, 50);
-    commit_metrics->Merge(other);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
-    ASSERT_EQ(80, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric"));
-    ASSERT_EQ(100, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric_2"));
-    ASSERT_EQ(200, counter);
-}
-
-}  // namespace paimon::test
diff --git a/src/paimon/core/operation/metrics/commit_stats.h b/src/paimon/core/operation/metrics/commit_stats.h
new file mode 100644
index 00000000..562ded5f
--- /dev/null
+++ b/src/paimon/core/operation/metrics/commit_stats.h
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/manifest_entry.h"
+
+namespace paimon {
+
+/// Statistics for a commit. Interface is aligned with Java CommitStats.
+class CommitStats {
+ public:
+    CommitStats(const std::vector& append_table_files,
+                const std::vector& append_changelog_files,
+                const std::vector& compact_table_files,
+                const std::vector& compact_changelog_files, int64_t commit_duration,
+                int32_t generated_snapshots, int32_t attempts, int64_t last_committed_snapshot_id) {
+        duration_ = commit_duration;
+        attempts_ = attempts;
+        table_files_appended_ = static_cast(append_table_files.size());
+        changelog_files_appended_ = static_cast(append_changelog_files.size());
+        changelog_files_compacted_ = static_cast(compact_changelog_files.size());
+        changelog_records_compacted_ = RowCounts(compact_changelog_files);
+        delta_records_compacted_ = RowCounts(compact_table_files);
+        changelog_records_appended_ = RowCounts(append_changelog_files);
+        delta_records_appended_ = RowCounts(append_table_files);
+        table_files_compacted_ = static_cast(compact_table_files.size());
+        generated_snapshots_ = generated_snapshots;
+        num_partitions_written_ = NumChangedPartitions({append_table_files, compact_table_files});
+        num_buckets_written_ = NumChangedBuckets({append_table_files, compact_table_files});
+        last_committed_snapshot_id_ = last_committed_snapshot_id;
+
+        std::vector added_table_files;
+        std::vector deleted_table_files;
+        for (const auto& entry : append_table_files) {
+            if (entry.Kind() == FileKind::Add()) {
+                added_table_files.push_back(entry);
+            } else if (entry.Kind() == FileKind::Delete()) {
+                deleted_table_files.push_back(entry);
+            }
+        }
+
+        std::vector compact_after_files;
+        std::vector compaction_input_files;
+        for (const auto& entry : compact_table_files) {
+            if (entry.Kind() == FileKind::Add()) {
+                compact_after_files.push_back(entry);
+            } else if (entry.Kind() == FileKind::Delete()) {
+                compaction_input_files.push_back(entry);
+            }
+        }
+
+        added_table_files.insert(added_table_files.end(), compact_after_files.begin(),
+                                 compact_after_files.end());
+        deleted_table_files.insert(deleted_table_files.end(), compaction_input_files.begin(),
+                                   compaction_input_files.end());
+
+        table_files_added_ = static_cast(added_table_files.size());
+        table_files_deleted_ = static_cast(deleted_table_files.size());
+        compaction_input_file_size_ = FileSizes(compaction_input_files);
+        compaction_output_file_size_ = FileSizes(compact_after_files);
+    }
+
+    int64_t GetTableFilesAdded() const {
+        return table_files_added_;
+    }
+    int64_t GetTableFilesDeleted() const {
+        return table_files_deleted_;
+    }
+    int64_t GetTableFilesAppended() const {
+        return table_files_appended_;
+    }
+    int64_t GetTableFilesCompacted() const {
+        return table_files_compacted_;
+    }
+    int64_t GetChangelogFilesAppended() const {
+        return changelog_files_appended_;
+    }
+    int64_t GetChangelogFilesCompacted() const {
+        return changelog_files_compacted_;
+    }
+    int64_t GetGeneratedSnapshots() const {
+        return generated_snapshots_;
+    }
+    int64_t GetDeltaRecordsAppended() const {
+        return delta_records_appended_;
+    }
+    int64_t GetChangelogRecordsAppended() const {
+        return changelog_records_appended_;
+    }
+    int64_t GetDeltaRecordsCompacted() const {
+        return delta_records_compacted_;
+    }
+    int64_t GetChangelogRecordsCompacted() const {
+        return changelog_records_compacted_;
+    }
+    int64_t GetNumPartitionsWritten() const {
+        return num_partitions_written_;
+    }
+    int64_t GetNumBucketsWritten() const {
+        return num_buckets_written_;
+    }
+    int64_t GetDuration() const {
+        return duration_;
+    }
+    int32_t GetAttempts() const {
+        return attempts_;
+    }
+    int64_t GetCompactionInputFileSize() const {
+        return compaction_input_file_size_;
+    }
+    int64_t GetCompactionOutputFileSize() const {
+        return compaction_output_file_size_;
+    }
+    int64_t GetLastCommittedSnapshotId() const {
+        return last_committed_snapshot_id_;
+    }
+
+    static int64_t NumChangedPartitions(const std::vector>& changes) {
+        std::unordered_set changed_partitions;
+        for (const auto& change : changes) {
+            for (const auto& entry : change) {
+                changed_partitions.insert(entry.Partition());
+            }
+        }
+        return static_cast(changed_partitions.size());
+    }
+
+    static int64_t NumChangedBuckets(const std::vector>& changes) {
+        std::unordered_map> changed_partition_buckets;
+        for (const auto& change : changes) {
+            for (const auto& entry : change) {
+                changed_partition_buckets[entry.Partition()].insert(entry.Bucket());
+            }
+        }
+
+        int64_t num_changed_buckets = 0;
+        for (const auto& [_, buckets] : changed_partition_buckets) {
+            num_changed_buckets += static_cast(buckets.size());
+        }
+        return num_changed_buckets;
+    }
+
+ private:
+    static int64_t RowCounts(const std::vector& files) {
+        int64_t row_count = 0;
+        for (const auto& entry : files) {
+            row_count += entry.File()->row_count;
+        }
+        return row_count;
+    }
+
+    static int64_t FileSizes(const std::vector& files) {
+        int64_t file_size = 0;
+        for (const auto& entry : files) {
+            file_size += entry.File()->file_size;
+        }
+        return file_size;
+    }
+
+ private:
+    int64_t duration_ = 0;
+    int32_t attempts_ = 0;
+    int64_t table_files_appended_ = 0;
+    int64_t table_files_added_ = 0;
+    int64_t table_files_deleted_ = 0;
+    int64_t changelog_files_appended_ = 0;
+    int64_t compaction_input_file_size_ = 0;
+    int64_t compaction_output_file_size_ = 0;
+    int64_t changelog_files_compacted_ = 0;
+    int64_t changelog_records_compacted_ = 0;
+    int64_t delta_records_compacted_ = 0;
+    int64_t changelog_records_appended_ = 0;
+    int64_t delta_records_appended_ = 0;
+    int64_t table_files_compacted_ = 0;
+    int64_t generated_snapshots_ = 0;
+    int64_t num_partitions_written_ = 0;
+    int64_t num_buckets_written_ = 0;
+    int64_t last_committed_snapshot_id_ = -1;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/operation/metrics/commit_stats_test.cpp b/src/paimon/core/operation/metrics/commit_stats_test.cpp
new file mode 100644
index 00000000..a18f88a6
--- /dev/null
+++ b/src/paimon/core/operation/metrics/commit_stats_test.cpp
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/operation/metrics/commit_stats.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+BinaryRow CreateIntRow(int32_t value) {
+    BinaryRow row(1);
+    BinaryRowWriter writer(&row, 20, GetDefaultPool().get());
+    writer.WriteInt(0, value);
+    writer.Complete();
+    return row;
+}
+
+ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucket,
+                          int64_t row_count, int64_t file_size, const std::string& file_name) {
+    BinaryRow part = CreateIntRow(partition);
+    auto file_meta = std::make_shared(
+        file_name, file_size, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(),
+        SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/0,
+        /*schema_id=*/1, /*level=*/0,
+        /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(0, 0),
+        /*delete_row_count=*/std::nullopt,
+        /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt,
+        /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta);
+}
+
+}  // namespace
+
+TEST(CommitStatsTest, TestCalcChangedPartitionsAndBuckets) {
+    std::vector files;
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1,
+                                /*row_count=*/201, /*file_size=*/11, "a1"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3,
+                                /*row_count=*/302, /*file_size=*/12, "a2"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1,
+                                /*row_count=*/202, /*file_size=*/13, "c1"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3,
+                                /*row_count=*/301, /*file_size=*/14, "c2"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1,
+                                /*row_count=*/203, /*file_size=*/15, "k1"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3,
+                                /*row_count=*/304, /*file_size=*/16, "k2"));
+    files.push_back(CreateEntry(FileKind::Delete(), /*partition=*/3, /*bucket=*/5,
+                                /*row_count=*/106, /*file_size=*/17, "k3"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1,
+                                /*row_count=*/205, /*file_size=*/18, "ck1"));
+    files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3,
+                                /*row_count=*/307, /*file_size=*/19, "ck2"));
+
+    EXPECT_EQ(3, CommitStats::NumChangedBuckets({files}));
+    EXPECT_EQ(3, CommitStats::NumChangedPartitions({files}));
+}
+
+TEST(CommitStatsTest, TestFailedAppendSnapshot) {
+    CommitStats stats(/*append_table_files=*/{}, /*append_changelog_files=*/{},
+                      /*compact_table_files=*/{}, /*compact_changelog_files=*/{},
+                      /*commit_duration=*/0, /*generated_snapshots=*/0, /*attempts=*/1,
+                      /*last_committed_snapshot_id=*/-1);
+
+    EXPECT_EQ(0, stats.GetTableFilesAdded());
+    EXPECT_EQ(0, stats.GetTableFilesDeleted());
+    EXPECT_EQ(0, stats.GetTableFilesAppended());
+    EXPECT_EQ(0, stats.GetTableFilesCompacted());
+    EXPECT_EQ(0, stats.GetChangelogFilesAppended());
+    EXPECT_EQ(0, stats.GetChangelogFilesCompacted());
+    EXPECT_EQ(0, stats.GetGeneratedSnapshots());
+    EXPECT_EQ(0, stats.GetDeltaRecordsAppended());
+    EXPECT_EQ(0, stats.GetChangelogRecordsAppended());
+    EXPECT_EQ(0, stats.GetDeltaRecordsCompacted());
+    EXPECT_EQ(0, stats.GetChangelogRecordsCompacted());
+    EXPECT_EQ(0, stats.GetNumPartitionsWritten());
+    EXPECT_EQ(0, stats.GetNumBucketsWritten());
+    EXPECT_EQ(0, stats.GetDuration());
+    EXPECT_EQ(1, stats.GetAttempts());
+    EXPECT_EQ(-1, stats.GetLastCommittedSnapshotId());
+}
+
+TEST(CommitStatsTest, TestSucceedAllSnapshot) {
+    std::vector append_data_files;
+    append_data_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 201, 1001, "a1"));
+    append_data_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 302, 1002, "a2"));
+
+    std::vector append_changelog_files;
+    append_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 202, 2001, "c1"));
+    append_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 301, 2002, "c2"));
+
+    std::vector compact_data_files;
+    compact_data_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 203, 3001, "k1"));
+    compact_data_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 304, 3002, "k2"));
+    compact_data_files.push_back(CreateEntry(FileKind::Delete(), 3, 5, 106, 3003, "k3"));
+
+    std::vector compact_changelog_files;
+    compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 205, 4001, "ck1"));
+    compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 307, 4002, "ck2"));
+
+    CommitStats stats(append_data_files, append_changelog_files, compact_data_files,
+                      compact_changelog_files,
+                      /*commit_duration=*/3000, /*generated_snapshots=*/2, /*attempts=*/2,
+                      /*last_committed_snapshot_id=*/10);
+
+    EXPECT_EQ(4, stats.GetTableFilesAdded());
+    EXPECT_EQ(1, stats.GetTableFilesDeleted());
+    EXPECT_EQ(2, stats.GetTableFilesAppended());
+    EXPECT_EQ(3, stats.GetTableFilesCompacted());
+    EXPECT_EQ(2, stats.GetChangelogFilesAppended());
+    EXPECT_EQ(2, stats.GetChangelogFilesCompacted());
+    EXPECT_EQ(2, stats.GetGeneratedSnapshots());
+    EXPECT_EQ(503, stats.GetDeltaRecordsAppended());
+    EXPECT_EQ(503, stats.GetChangelogRecordsAppended());
+    EXPECT_EQ(613, stats.GetDeltaRecordsCompacted());
+    EXPECT_EQ(512, stats.GetChangelogRecordsCompacted());
+    EXPECT_EQ(3, stats.GetNumPartitionsWritten());
+    EXPECT_EQ(3, stats.GetNumBucketsWritten());
+    EXPECT_EQ(3000, stats.GetDuration());
+    EXPECT_EQ(2, stats.GetAttempts());
+    EXPECT_EQ(10, stats.GetLastCommittedSnapshotId());
+    EXPECT_EQ(3003, stats.GetCompactionInputFileSize());
+    EXPECT_EQ(6003, stats.GetCompactionOutputFileSize());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/operation/metrics/compaction_metrics.h b/src/paimon/core/operation/metrics/compaction_metrics.h
index aa48a5e4..24a7169f 100644
--- a/src/paimon/core/operation/metrics/compaction_metrics.h
+++ b/src/paimon/core/operation/metrics/compaction_metrics.h
@@ -52,16 +52,16 @@ class CompactionMetrics {
             : metrics_(metrics), partition_(partition), bucket_(bucket) {}
 
         void ReportLevel0FileCount(int64_t count) {
-            level0_file_count_ = count;
+            level0_file_count_.store(count, std::memory_order_relaxed);
         }
         void ReportCompactionInputSize(int64_t bytes) {
-            compaction_input_size_ = bytes;
+            compaction_input_size_.store(bytes, std::memory_order_relaxed);
         }
         void ReportCompactionOutputSize(int64_t bytes) {
-            compaction_output_size_ = bytes;
+            compaction_output_size_.store(bytes, std::memory_order_relaxed);
         }
         void ReportTotalFileSize(int64_t bytes) {
-            total_file_size_ = bytes;
+            total_file_size_.store(bytes, std::memory_order_relaxed);
         }
         void ReportCompactionTime(int64_t time) {
             metrics_->ReportCompactionTime(time);
@@ -84,19 +84,19 @@ class CompactionMetrics {
         }
 
         int64_t Level0FileCount() const {
-            return level0_file_count_;
+            return level0_file_count_.load(std::memory_order_relaxed);
         }
 
         int64_t CompactionInputSize() const {
-            return compaction_input_size_;
+            return compaction_input_size_.load(std::memory_order_relaxed);
         }
 
         int64_t CompactionOutputSize() const {
-            return compaction_output_size_;
+            return compaction_output_size_.load(std::memory_order_relaxed);
         }
 
         int64_t TotalFileSize() const {
-            return total_file_size_;
+            return total_file_size_.load(std::memory_order_relaxed);
         }
 
      private:
@@ -105,10 +105,10 @@ class CompactionMetrics {
         int32_t bucket_;
 
         // Data fields for metrics.
-        int64_t level0_file_count_ = 0;
-        int64_t compaction_input_size_ = 0;
-        int64_t compaction_output_size_ = 0;
-        int64_t total_file_size_ = 0;
+        std::atomic level0_file_count_ = {0};
+        std::atomic compaction_input_size_ = {0};
+        std::atomic compaction_output_size_ = {0};
+        std::atomic total_file_size_ = {0};
     };
 
     std::shared_ptr CreateReporter(const BinaryRow& partition, int32_t bucket) {
diff --git a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp
index 56919dd9..b8f27424 100644
--- a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp
+++ b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/operation/metrics/compaction_metrics.h"
 
 #include 
+#include 
 
 #include "gtest/gtest.h"
 #include "paimon/testing/utils/testharness.h"
@@ -149,4 +150,38 @@ TEST(CompactionMetricsTest, TestCompactionTimeWindow) {
     EXPECT_DOUBLE_EQ(60.5, avg_time);
 }
 
+TEST(CompactionMetricsTest, TestConcurrentReporterUpdateAndMetricsSnapshot) {
+    CompactionMetrics metrics;
+    std::shared_ptr reporter =
+        metrics.CreateReporter(BinaryRow::EmptyRow(), 0);
+
+    constexpr int64_t kLastValue = 9999;
+    std::thread writer([&reporter]() {
+        for (int64_t value = 0; value <= kLastValue; ++value) {
+            reporter->ReportLevel0FileCount(value);
+            reporter->ReportCompactionInputSize(value);
+            reporter->ReportCompactionOutputSize(value);
+            reporter->ReportTotalFileSize(value);
+        }
+    });
+    for (int64_t i = 0; i <= kLastValue; ++i) {
+        metrics.GetMetrics();
+    }
+    writer.join();
+
+    std::shared_ptr snapshot = metrics.GetMetrics();
+    ASSERT_OK_AND_ASSIGN(double level0_file_count,
+                         snapshot->GetGauge(CompactionMetrics::MAX_LEVEL0_FILE_COUNT));
+    ASSERT_OK_AND_ASSIGN(double compaction_input_size,
+                         snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_INPUT_SIZE));
+    ASSERT_OK_AND_ASSIGN(double compaction_output_size,
+                         snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_OUTPUT_SIZE));
+    ASSERT_OK_AND_ASSIGN(double total_file_size,
+                         snapshot->GetGauge(CompactionMetrics::MAX_TOTAL_FILE_SIZE));
+    ASSERT_DOUBLE_EQ(static_cast(kLastValue), level0_file_count);
+    ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_input_size);
+    ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_output_size);
+    ASSERT_DOUBLE_EQ(static_cast(kLastValue), total_file_size);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp
index 0b1df483..ac69d5a3 100644
--- a/src/paimon/core/operation/orphan_files_cleaner.cpp
+++ b/src/paimon/core/operation/orphan_files_cleaner.cpp
@@ -197,7 +197,8 @@ Result> OrphanFilesCleaner::Create(
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr manifest_list,
         ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(),
-                             options.GetManifestCompression(), path_factory, ctx->GetMemoryPool()));
+                             options.GetManifestCompression(), path_factory, options.GetCache(),
+                             ctx->GetMemoryPool()));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr partition_schema,
         FieldMapping::GetPartitionSchema(arrow_schema, table_schema.value()->PartitionKeys()));
diff --git a/src/paimon/core/operation/orphan_files_cleaner_test.cpp b/src/paimon/core/operation/orphan_files_cleaner_test.cpp
index 37731049..7749b692 100644
--- a/src/paimon/core/operation/orphan_files_cleaner_test.cpp
+++ b/src/paimon/core/operation/orphan_files_cleaner_test.cpp
@@ -230,7 +230,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithChangelog) {
   "commitIdentifier" : 9223372036854775807,
   "commitKind" : "APPEND",
   "timeMillis" : 1721615035363,
-  "logOffsets" : { },
   "totalRecordCount" : 11,
   "deltaRecordCount" : 1,
   "changelogRecordCount" : 0
@@ -263,7 +262,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithIndexManifest) {
   "commitIdentifier" : 9223372036854775807,
   "commitKind" : "APPEND",
   "timeMillis" : 1721615035363,
-  "logOffsets" : { },
   "totalRecordCount" : 11,
   "deltaRecordCount" : 1,
   "changelogRecordCount" : 0
diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp
index c06fb055..eabe8426 100644
--- a/src/paimon/core/operation/raw_file_split_read.cpp
+++ b/src/paimon/core/operation/raw_file_split_read.cpp
@@ -83,7 +83,8 @@ Result> RawFileSplitRead::CreateReader(
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory,
-                             /*row_ranges=*/{}, data_file_path_factory));
+                             /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     auto raw_readers =
         ObjectUtils::MoveVector>(std::move(raw_file_readers));
@@ -98,7 +99,8 @@ Result> RawFileSplitRead::CreateReader(
     const std::vector>& data_files,
     const std::vector>& deletion_files) {
     auto dv_factory = DeletionVector::CreateFactory(
-        options_.GetFileSystem(), CreateDeletionFileMap(data_files, deletion_files), pool_);
+        options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files),
+        pool_);
     return CreateReader(partition, bucket, data_files, dv_factory);
 }
 
diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h
index 3aa1ada6..1580cd84 100644
--- a/src/paimon/core/operation/raw_file_split_read.h
+++ b/src/paimon/core/operation/raw_file_split_read.h
@@ -55,7 +55,8 @@ struct DeletionFile;
 /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader)
 /// ->ConcatBatchReader across
 /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader)
-/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
+/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader)
+/// ->(PrefetchFileBatchReader)->FormatReader
 
 class RawFileSplitRead : public AbstractSplitRead {
  public:
diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp
index e108a10f..569f8dfd 100644
--- a/src/paimon/core/operation/raw_file_split_read_test.cpp
+++ b/src/paimon/core/operation/raw_file_split_read_test.cpp
@@ -135,7 +135,7 @@ class RawFileSplitReadTest : public ::testing::Test {
                            "/orc/multi_partition_append_table.db/"
                            "multi_partition_append_table";
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema(read_schema->field_names());
+        context_builder.SetReadFieldNames(read_schema->field_names());
         ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, context_builder.Finish());
         SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
         ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
@@ -159,9 +159,10 @@ class RawFileSplitReadTest : public ::testing::Test {
                 core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
                 external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
                 pool_));
-        auto split_read =
-            std::make_unique(path_factory, std::move(internal_context), pool_,
-                                               CreateDefaultExecutor(/*thread_count=*/2));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr executor,
+                             CreateDefaultExecutor(/*thread_count=*/2));
+        auto split_read = std::make_unique(
+            path_factory, std::move(internal_context), pool_, executor);
 
         std::vector> batch_readers;
         batch_readers.reserve(data_splits.size());
@@ -404,9 +405,10 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) {
             external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
             pool_));
 
-    auto split_read =
-        std::make_unique(path_factory, std::move(internal_context), pool_,
-                                           CreateDefaultExecutor(/*thread_count=*/2));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr executor,
+                         CreateDefaultExecutor(/*thread_count=*/2));
+    auto split_read = std::make_unique(path_factory, std::move(internal_context),
+                                                         pool_, executor);
     DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()),
                                    /*bucket=*/0, /*bucket_path=*/
                                    paimon::test::GetDataDir() +
@@ -428,16 +430,17 @@ TEST_F(RawFileSplitReadTest, TestMatch) {
     std::string path = paimon::test::GetDataDir() +
                        "/orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f0", "f1", "f2", "f3"});
+    context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"});
     ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, context_builder.Finish());
     SchemaManager schema_manager(std::make_shared(), read_context->GetPath());
     ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
     ASSERT_OK_AND_ASSIGN(std::shared_ptr internal_context,
                          InternalReadContext::Create(std::move(read_context), table_schema,
                                                      table_schema->Options()));
-    auto split_read =
-        std::make_unique(/*path_factory=*/nullptr, std::move(internal_context),
-                                           pool_, CreateDefaultExecutor(/*thread_count=*/2));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr executor,
+                         CreateDefaultExecutor(/*thread_count=*/2));
+    auto split_read = std::make_unique(
+        /*path_factory=*/nullptr, std::move(internal_context), pool_, executor);
     auto create_data_split = [this](bool is_streaming,
                                     bool raw_convertible) -> std::shared_ptr {
         auto meta = std::make_shared(
diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp
index 81fe2365..ad7d1e3a 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -20,6 +20,8 @@
 
 #include 
 
+#include "arrow/c/abi.h"
+#include "arrow/c/bridge.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/core/utils/branch_manager.h"
 #include "paimon/executor.h"
@@ -30,19 +32,20 @@ namespace paimon {
 class Predicate;
 
 ReadContext::ReadContext(
-    const std::string& path, const std::string& branch, const std::vector& read_schema,
-    const std::vector& read_field_ids, const std::shared_ptr& predicate,
-    bool enable_predicate_filter, bool enable_prefetch, uint32_t prefetch_batch_count,
-    uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch,
-    uint32_t row_to_batch_thread_number, const std::optional& table_schema,
-    const std::shared_ptr& memory_pool, const std::shared_ptr& executor,
+    const std::string& path, const std::string& branch,
+    const std::vector& read_field_names, const std::vector& read_field_ids,
+    const std::shared_ptr& predicate, bool enable_predicate_filter, bool enable_prefetch,
+    uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num,
+    bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number,
+    const std::optional& table_schema, const std::shared_ptr& memory_pool,
+    const std::shared_ptr& executor,
     const std::shared_ptr& specific_file_system,
     const std::map& fs_scheme_to_identifier_map,
     const std::map& options, PrefetchCacheMode prefetch_cache_mode,
-    const CacheConfig& cache_config)
+    const CacheConfig& cache_config, const std::shared_ptr& cache)
     : path_(path),
       branch_(branch),
-      read_schema_(read_schema),
+      read_field_names_(read_field_names),
       read_field_ids_(read_field_ids),
       predicate_(predicate),
       enable_predicate_filter_(enable_predicate_filter),
@@ -58,9 +61,26 @@ ReadContext::ReadContext(
       fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map),
       options_(options),
       prefetch_cache_mode_(prefetch_cache_mode),
-      cache_config_(cache_config) {}
+      cache_config_(cache_config),
+      cache_(cache) {}
 
-ReadContext::~ReadContext() = default;
+ReadContext::~ReadContext() {
+    if (read_schema_ && read_schema_->release) {
+        read_schema_->release(read_schema_.get());
+    }
+}
+
+void ReadContext::SetReadSchema(std::unique_ptr schema) {
+    if (schema && schema->release) {
+        if (schema.get() == read_schema_.get()) {
+            return;
+        }
+        if (read_schema_ && read_schema_->release) {
+            read_schema_->release(read_schema_.get());
+        }
+        read_schema_ = std::move(schema);
+    }
+}
 
 class ReadContextBuilder::Impl {
  public:
@@ -69,6 +89,7 @@ class ReadContextBuilder::Impl {
         branch_ = BranchManager::DEFAULT_MAIN_BRANCH;
         read_field_names_.clear();
         read_field_ids_.clear();
+        read_schema_.reset();
         fs_scheme_to_identifier_map_.clear();
         options_.clear();
         predicate_.reset();
@@ -84,6 +105,7 @@ class ReadContextBuilder::Impl {
         executor_.reset();
         specific_file_system_.reset();
         cache_config_ = CacheConfig();
+        cache_.reset();
     }
 
  private:
@@ -91,6 +113,7 @@ class ReadContextBuilder::Impl {
     std::string branch_ = BranchManager::DEFAULT_MAIN_BRANCH;
     std::vector read_field_names_;
     std::vector read_field_ids_;
+    std::unique_ptr read_schema_;
     std::map fs_scheme_to_identifier_map_;
     std::map options_;
     std::shared_ptr predicate_;
@@ -106,6 +129,7 @@ class ReadContextBuilder::Impl {
     std::shared_ptr specific_file_system_;
     PrefetchCacheMode prefetch_cache_mode_ = PrefetchCacheMode::ALWAYS;
     CacheConfig cache_config_;
+    std::shared_ptr cache_;
 };
 
 ReadContextBuilder::ReadContextBuilder(const std::string& path)
@@ -129,7 +153,7 @@ ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& read_field_names) {
     impl_->read_field_names_ = read_field_names;
     return *this;
@@ -141,6 +165,13 @@ ReadContextBuilder& ReadContextBuilder::SetReadFieldIds(
     return *this;
 }
 
+ReadContextBuilder& ReadContextBuilder::SetReadSchema(std::unique_ptr read_schema) {
+    if (read_schema && read_schema->release) {
+        impl_->read_schema_ = std::move(read_schema);
+    }
+    return *this;
+}
+
 ReadContextBuilder& ReadContextBuilder::SetPredicate(const std::shared_ptr& predicate) {
     impl_->predicate_ = predicate;
     return *this;
@@ -219,11 +250,19 @@ ReadContextBuilder& ReadContextBuilder::WithCacheConfig(const CacheConfig& cache
     return *this;
 }
 
+ReadContextBuilder& ReadContextBuilder::WithCache(const std::shared_ptr& cache) {
+    impl_->cache_ = cache;
+    return *this;
+}
+
 Result> ReadContextBuilder::Finish() {
     PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_));
     if (impl_->path_.empty()) {
         return Status::Invalid("cannot read with empty table path");
     }
+    if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) {
+        return Status::Invalid("prefetch max parallel num should be greater than 0");
+    }
     if (impl_->enable_prefetch_ && impl_->prefetch_batch_count_ <= 0) {
         return Status::Invalid("prefetch batch count should be greater than 0");
     }
@@ -232,10 +271,14 @@ Result> ReadContextBuilder::Finish() {
         return Status::Invalid(
             "prefetch batch count should be greater than or equal to prefetch max parallel num");
     }
+    if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) {
+        return Status::Invalid(
+            "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together");
+    }
     if (!impl_->executor_) {
         // If the user do not set executor, create default executor by prefetch batch count
         uint32_t thread_count = impl_->enable_prefetch_ ? impl_->prefetch_max_parallel_num_ : 1;
-        impl_->executor_ = CreateDefaultExecutor(thread_count);
+        PAIMON_ASSIGN_OR_RAISE(impl_->executor_, CreateDefaultExecutor(thread_count));
     }
 
     if (impl_->enable_multi_thread_row_to_batch_ && impl_->row_to_batch_thread_number_ <= 0) {
@@ -248,7 +291,10 @@ Result> ReadContextBuilder::Finish() {
         impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_,
         impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_,
         impl_->fs_scheme_to_identifier_map_, impl_->options_, impl_->prefetch_cache_mode_,
-        impl_->cache_config_);
+        impl_->cache_config_, impl_->cache_);
+    if (impl_->read_schema_ && impl_->read_schema_->release) {
+        ctx->SetReadSchema(std::move(impl_->read_schema_));
+    }
     impl_->Reset();
     return ctx;
 }
diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp
index dc79b69e..8d5a78e1 100644
--- a/src/paimon/core/operation/read_context_test.cpp
+++ b/src/paimon/core/operation/read_context_test.cpp
@@ -20,7 +20,10 @@
 
 #include 
 
+#include "arrow/c/bridge.h"
+#include "arrow/type.h"
 #include "gtest/gtest.h"
+#include "paimon/common/io/cache/lru_cache.h"
 #include "paimon/defs.h"
 #include "paimon/executor.h"
 #include "paimon/memory/memory_pool.h"
@@ -36,7 +39,7 @@ TEST(ReadContextTest, TestDefaultValue) {
     ASSERT_EQ(ctx->GetPath(), "table_root_path");
     ASSERT_TRUE(ctx->GetMemoryPool());
     ASSERT_TRUE(ctx->GetExecutor());
-    ASSERT_TRUE(ctx->GetReadSchema().empty());
+    ASSERT_TRUE(ctx->GetReadFieldNames().empty());
     ASSERT_TRUE(ctx->GetReadFieldIds().empty());
     ASSERT_TRUE(ctx->GetOptions().empty());
     ASSERT_FALSE(ctx->GetPredicate());
@@ -60,7 +63,7 @@ TEST(ReadContextTest, TestSetContent) {
                              /*hole_size_limit=*/128, /*pre_buffer_limit=*/2048);
 
     builder.AddOption("key", "value");
-    builder.SetReadSchema({"f1", "f2"});
+    builder.SetReadFieldNames({"f1", "f2"});
     builder.SetReadFieldIds({0, 1});
     auto predicate =
         PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f1", FieldType::INT);
@@ -77,16 +80,17 @@ TEST(ReadContextTest, TestSetContent) {
     builder.SetTableSchema("table-schema-json");
     builder.WithBranch("rt");
     builder.WithCacheConfig(cache_config);
-    builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
     auto fs = std::make_shared();
     builder.WithFileSystem(fs);
+    auto manifest_cache = std::make_shared(1024);
+    builder.WithCache(manifest_cache);
     ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
 
     // test result
     ASSERT_EQ(ctx->GetPath(), "table_root_path");
     ASSERT_TRUE(ctx->GetMemoryPool());
     ASSERT_TRUE(ctx->GetExecutor());
-    ASSERT_EQ(ctx->GetReadSchema(), std::vector({"f1", "f2"}));
+    ASSERT_EQ(ctx->GetReadFieldNames(), std::vector({"f1", "f2"}));
     ASSERT_EQ(ctx->GetReadFieldIds(), std::vector({0, 1}));
     ASSERT_EQ(*predicate, *(ctx->GetPredicate()));
     ASSERT_TRUE(ctx->EnablePredicateFilter());
@@ -105,11 +109,11 @@ TEST(ReadContextTest, TestSetContent) {
     ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit());
     ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit());
     ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit());
-    std::map expected_fs_map = {{"file", "local"}};
-    ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap());
+    ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty());
     std::map expected_options = {{"key", "value"}};
     ASSERT_EQ(expected_options, ctx->GetOptions());
     ASSERT_EQ(ctx->GetSpecificFileSystem(), fs);
+    ASSERT_TRUE(ctx->GetCache());
 }
 
 TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) {
@@ -123,4 +127,58 @@ TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) {
     ASSERT_EQ(expected_options, ctx->GetOptions());
 }
 
+TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) {
+    ReadContextBuilder builder("table_root_path");
+    auto fs = std::make_shared();
+    builder.WithFileSystem(fs);
+    builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
+    ASSERT_NOK_WITH_MSG(
+        builder.Finish(),
+        "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together");
+}
+
+TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) {
+    ReadContextBuilder builder("table_root_path");
+    builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
+    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
+    std::map expected_fs_map = {{"file", "local"}};
+    ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap());
+    ASSERT_FALSE(ctx->GetSpecificFileSystem());
+}
+
+TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) {
+    ReadContextBuilder builder("table_root_path");
+    builder.EnablePrefetch(true);
+    builder.SetPrefetchMaxParallelNum(0);
+    ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0");
+}
+
+TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) {
+    auto projected_schema = arrow::schema({arrow::field("f0", arrow::utf8())});
+    auto c_schema = std::make_unique();
+    auto* c_schema_raw = c_schema.get();
+    ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok());
+
+    {
+        ReadContextBuilder builder("table_root_path");
+        builder.SetReadSchema(std::move(c_schema));
+        ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
+        ASSERT_TRUE(ctx->HasReadSchema());
+        ASSERT_EQ(ctx->GetReadSchema(), c_schema_raw);
+    }
+
+    ASSERT_EQ(c_schema, nullptr);
+}
+
+TEST(ReadContextTest, TestSetInvalidReadSchemaIgnored) {
+    auto invalid_schema = std::make_unique();
+
+    ReadContextBuilder builder("table_root_path");
+    builder.SetReadSchema(std::move(invalid_schema));
+    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
+
+    ASSERT_FALSE(ctx->HasReadSchema());
+    ASSERT_EQ(ctx->GetReadSchema(), nullptr);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp
index 794480ed..b59fa098 100644
--- a/src/paimon/core/operation/scan_context.cpp
+++ b/src/paimon/core/operation/scan_context.cpp
@@ -35,7 +35,9 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode,
                          const std::shared_ptr& memory_pool,
                          const std::shared_ptr& executor,
                          const std::shared_ptr& specific_file_system,
-                         const std::map& options)
+                         const std::optional& table_schema,
+                         const std::map& options,
+                         const std::shared_ptr& cache)
     : path_(path),
       is_streaming_mode_(is_streaming_mode),
       limit_(limit),
@@ -44,7 +46,9 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode,
       memory_pool_(memory_pool),
       executor_(executor),
       specific_file_system_(specific_file_system),
-      options_(options) {}
+      table_schema_(table_schema),
+      options_(options),
+      cache_(cache) {}
 
 ScanContext::~ScanContext() = default;
 
@@ -62,7 +66,9 @@ class ScanContextBuilder::Impl {
         memory_pool_ = GetDefaultPool();
         executor_ = CreateDefaultExecutor();
         specific_file_system_.reset();
+        table_schema_ = std::nullopt;
         options_.clear();
+        cache_.reset();
     }
 
  private:
@@ -76,7 +82,9 @@ class ScanContextBuilder::Impl {
     std::shared_ptr memory_pool_ = GetDefaultPool();
     std::shared_ptr executor_ = CreateDefaultExecutor();
     std::shared_ptr specific_file_system_;
+    std::optional table_schema_;
     std::map options_;
+    std::shared_ptr cache_;
 };
 
 ScanContextBuilder::ScanContextBuilder(const std::string& path)
@@ -145,6 +153,16 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem(
     return *this;
 }
 
+ScanContextBuilder& ScanContextBuilder::SetTableSchema(const std::string& table_schema) {
+    impl_->table_schema_ = table_schema;
+    return *this;
+}
+
+ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& cache) {
+    impl_->cache_ = cache;
+    return *this;
+}
+
 Result> ScanContextBuilder::Finish() {
     PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_));
     if (impl_->path_.empty()) {
@@ -155,7 +173,7 @@ Result> ScanContextBuilder::Finish() {
         std::make_shared(impl_->predicates_, impl_->partition_filters_,
                                      impl_->bucket_filter_),
         impl_->global_index_result_, impl_->memory_pool_, impl_->executor_,
-        impl_->specific_file_system_, impl_->options_);
+        impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_);
     impl_->Reset();
     return ctx;
 }
diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp
index dd3914d6..6c86c049 100644
--- a/src/paimon/core/operation/scan_context_test.cpp
+++ b/src/paimon/core/operation/scan_context_test.cpp
@@ -19,6 +19,7 @@
 #include "paimon/scan_context.h"
 
 #include "gtest/gtest.h"
+#include "paimon/common/io/cache/lru_cache.h"
 #include "paimon/defs.h"
 #include "paimon/executor.h"
 #include "paimon/global_index/bitmap_global_index_result.h"
@@ -67,6 +68,9 @@ TEST(ScanContextTest, TestSetContent) {
     builder.WithExecutor(executor);
     auto fs = std::make_shared();
     builder.WithFileSystem(fs);
+    builder.SetTableSchema("table-schema-json");
+    auto manifest_cache = std::make_shared(1024);
+    builder.WithCache(manifest_cache);
     ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
     ASSERT_EQ(ctx->GetPath(), "table_root_path");
     ASSERT_TRUE(ctx->IsStreamingMode());
@@ -78,9 +82,12 @@ TEST(ScanContextTest, TestSetContent) {
     ASSERT_EQ("{1,2,4,5}", ctx->GetGlobalIndexResult()->ToString());
     ASSERT_EQ(memory_pool, ctx->GetMemoryPool());
     ASSERT_EQ(executor, ctx->GetExecutor());
+    ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value());
+    ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value());
     std::map expected_options = {{"key", "value"}};
     ASSERT_EQ(expected_options, ctx->GetOptions());
     ASSERT_EQ(fs, ctx->GetSpecificFileSystem());
+    ASSERT_TRUE(ctx->GetCache());
 }
 
 TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) {
diff --git a/src/paimon/core/options/map_shared_shredding_column_placement_policy.h b/src/paimon/core/options/map_shared_shredding_column_placement_policy.h
new file mode 100644
index 00000000..040c5b32
--- /dev/null
+++ b/src/paimon/core/options/map_shared_shredding_column_placement_policy.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+namespace paimon {
+
+/// Specifies how shared-shredding MAP fields choose physical columns.
+enum class MapSharedShreddingColumnPlacementPolicy {
+    /// Keep the key order from each input MAP row and place the first K keys into columns 0..K-1.
+    /// Use this when the input order is meaningful and should directly decide column placement.
+    /// Example:
+    ///   K=2
+    ///   row [c, a, b] -> ordered [c, a, b] -> columns [c, a], overflow [b]
+    ///   row [a, b, c] -> ordered [a, b, c] -> columns [a, b], overflow [c]
+    PLAIN = 0,
+    /// Use the shared-shredding metadata order before placing the first K keys into columns 0..K-1.
+    /// Use this when each row should choose leading-column keys by a deterministic metadata
+    /// order instead of the input MAP entry order, and no cross-row history should be used.
+    /// Example:
+    ///   K=2, metadata order [b, c, a]
+    ///   row [c, a, b] -> ordered [b, c, a] -> columns [b, c], overflow [a]
+    ///   row [c, a]    -> ordered [c, a]    -> columns [c, a], overflow []
+    /// This does not reserve key-to-column mappings across rows.
+    SEQUENTIAL = 1,
+    /// Reuse columns for recently seen keys when possible; otherwise choose an empty column first,
+    /// then the least-recently-used physical column.
+    /// Use this when hot keys are likely to appear repeatedly across nearby rows and should stay
+    /// in physical columns when possible.
+    /// Example:
+    ///   K=3
+    ///   row [a, b, d]    -> ordered [a, b, d]    -> columns [a, b, d], overflow []
+    ///   recently used columns are now [a, b, d]
+    ///   row [d, c, a, b] -> ordered [a, b, c, d] -> columns [a, b, d], overflow [c]
+    /// If a key has already been evicted, it is treated like a new key when it appears again.
+    LRU = 2
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/options/map_storage_layout.h b/src/paimon/core/options/map_storage_layout.h
new file mode 100644
index 00000000..5f27cdb5
--- /dev/null
+++ b/src/paimon/core/options/map_storage_layout.h
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+namespace paimon {
+/// Specifies the physical storage layout for MAP columns.
+enum class MapStorageLayout {
+    /// Default KV-array storage (keys array + values array).
+    DEFAULT = 0,
+    /// Shared-shredding layout: K reusable typed columns with per-row field mapping.
+    SHARED_SHREDDING = 1
+};
+}  // namespace paimon
diff --git a/src/paimon/core/postpone/postpone_bucket_file_store_write.h b/src/paimon/core/postpone/postpone_bucket_file_store_write.h
index d1c2ef41..1ca65d28 100644
--- a/src/paimon/core/postpone/postpone_bucket_file_store_write.h
+++ b/src/paimon/core/postpone/postpone_bucket_file_store_write.h
@@ -127,10 +127,11 @@ class PostponeBucketFileStoreWrite : public AbstractFileStoreWrite {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr data_file_path_factory,
             file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
-        auto writer =
-            std::make_shared(trimmed_primary_keys, data_file_path_factory,
-                                                   table_schema_->Id(), schema_, options_, pool_);
-        return std::shared_ptr(writer);
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr writer,
+            PostponeBucketWriter::Create(trimmed_primary_keys, data_file_path_factory,
+                                         table_schema_->Id(), schema_, options_, pool_));
+        return writer;
     }
 
     Result> CreateFileStoreScan(
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp
index 543f83d7..fb27d001 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp
@@ -32,6 +32,7 @@
 #include "arrow/scalar.h"
 #include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
+#include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
@@ -42,8 +43,8 @@
 #include "paimon/core/io/compact_increment.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/io/data_increment.h"
-#include "paimon/core/io/key_value_data_file_writer.h"
-#include "paimon/core/io/single_file_writer.h"
+#include "paimon/core/io/key_value_data_file_writer_factory.h"
+#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/utils/commit_increment.h"
 #include "paimon/format/file_format.h"
@@ -54,10 +55,36 @@ namespace paimon {
 class InternalRow;
 class MemoryPool;
 
+namespace {
+
+std::shared_ptr BuildPostponeBucketWriteSchema(
+    const std::shared_ptr& value_schema) {
+    arrow::FieldVector target_fields;
+    target_fields.push_back(
+        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
+    target_fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()));
+    target_fields.insert(target_fields.end(), value_schema->fields().begin(),
+                         value_schema->fields().end());
+    return arrow::schema(target_fields);
+}
+
+}  // namespace
+
+Result> PostponeBucketWriter::Create(
+    const std::vector& trimmed_primary_keys,
+    const std::shared_ptr& path_factory, int64_t schema_id,
+    const std::shared_ptr& value_schema, const CoreOptions& options,
+    const std::shared_ptr& pool) {
+    auto write_schema = BuildPostponeBucketWriteSchema(value_schema);
+    return std::unique_ptr(new PostponeBucketWriter(
+        trimmed_primary_keys, path_factory, schema_id, value_schema, write_schema, options, pool));
+}
+
 PostponeBucketWriter::PostponeBucketWriter(const std::vector& trimmed_primary_keys,
                                            const std::shared_ptr& path_factory,
                                            int64_t schema_id,
                                            const std::shared_ptr& value_schema,
+                                           const std::shared_ptr& write_schema,
                                            const CoreOptions& options,
                                            const std::shared_ptr& pool)
     : pool_(pool),
@@ -67,15 +94,8 @@ PostponeBucketWriter::PostponeBucketWriter(const std::vector& trimm
       path_factory_(path_factory),
       schema_id_(schema_id),
       value_type_(arrow::struct_(value_schema->fields())),
-      metrics_(std::make_shared()) {
-    arrow::FieldVector target_fields;
-    target_fields.push_back(
-        DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
-    target_fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()));
-    target_fields.insert(target_fields.end(), value_schema->fields().begin(),
-                         value_schema->fields().end());
-    write_schema_ = arrow::schema(target_fields);
-}
+      write_schema_(write_schema),
+      metrics_(std::make_shared()) {}
 
 Status PostponeBucketWriter::Write(std::unique_ptr&& moved_batch) {
     if (moved_batch->GetData()->length == 0) {
@@ -116,7 +136,10 @@ Status PostponeBucketWriter::Write(std::unique_ptr&& moved_batch) {
 
     // write KeyValueBatch to RollingFileWriter
     if (!writer_) {
-        writer_ = CreateRollingRowWriter();
+        std::unique_ptr>>
+            rolling_writer;
+        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
+        writer_ = std::move(rolling_writer);
     }
     PAIMON_RETURN_NOT_OK(writer_->Write(std::move(key_value_batch)));
     return Status::OK();
@@ -238,32 +261,25 @@ PostponeBucketWriter::PrepareMinMaxKey(
                                       value_struct_array->length() - 1));
 }
 
-std::unique_ptr>>
+Result>>>
 PostponeBucketWriter::CreateRollingRowWriter() const {
-    auto create_file_writer = [&]()
-        -> Result>>> {
-        ::ArrowSchema arrow_schema;
-        ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); });
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema));
-        auto format = options_.GetWriteFileFormat(/*level=*/0);
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr writer_builder,
-            format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize()));
-        writer_builder->WithMemoryPool(pool_);
-        auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status {
-            ArrowArrayMove(key_value_batch.batch.get(), array);
-            return Status::OK();
-        };
-        auto writer = std::make_unique(
-            options_.GetWriteFileCompression(0), converter, schema_id_, /*level=*/0,
-            FileSource::Append(), trimmed_primary_keys_, /*stats_extractor=*/nullptr, write_schema_,
-            path_factory_->IsExternalPath(), pool_);
-        PAIMON_RETURN_NOT_OK(
-            writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder));
-        return writer;
-    };
+    std::shared_ptr>> factory;
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
+            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory,
+            pool_);
+    } else {
+        factory = std::make_shared(
+            options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
+            trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_);
+    }
     return std::make_unique>>(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer);
+        options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
+        factory);
 }
 
 Status PostponeBucketWriter::Flush() {
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.h b/src/paimon/core/postpone/postpone_bucket_writer.h
index 9da8f32c..f61c9887 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.h
+++ b/src/paimon/core/postpone/postpone_bucket_writer.h
@@ -51,10 +51,11 @@ class Metrics;
 
 class PostponeBucketWriter : public BatchWriter {
  public:
-    PostponeBucketWriter(const std::vector& trimmed_primary_keys,
-                         const std::shared_ptr& path_factory,
-                         int64_t schema_id, const std::shared_ptr& value_schema,
-                         const CoreOptions& options, const std::shared_ptr& pool);
+    static Result> Create(
+        const std::vector& trimmed_primary_keys,
+        const std::shared_ptr& path_factory, int64_t schema_id,
+        const std::shared_ptr& value_schema, const CoreOptions& options,
+        const std::shared_ptr& pool);
 
     ~PostponeBucketWriter() override {
         [[maybe_unused]] auto status = DoClose();
@@ -121,9 +122,15 @@ class PostponeBucketWriter : public BatchWriter {
     Status Flush();
     Result DrainIncrement();
 
-    std::unique_ptr>>
+    Result>>>
     CreateRollingRowWriter() const;
 
+    PostponeBucketWriter(const std::vector& trimmed_primary_keys,
+                         const std::shared_ptr& path_factory,
+                         int64_t schema_id, const std::shared_ptr& value_schema,
+                         const std::shared_ptr& write_schema,
+                         const CoreOptions& options, const std::shared_ptr& pool);
+
  private:
     std::shared_ptr pool_;
     std::unique_ptr arrow_pool_;
diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
index 4e2d211a..a629a6bd 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
@@ -29,6 +29,8 @@
 #include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/common/data/data_define.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/common/fs/external_path_provider.h"
 #include "paimon/common/table/special_fields.h"
@@ -110,6 +112,32 @@ class PostponeBucketWriterTest : public ::testing::Test,
                                                           << expected_array->ToString();
     }
 
+    void CheckShreddingFileSchema(const std::string& file_format_str,
+                                  const std::string& data_file_name,
+                                  const std::shared_ptr& expected_schema,
+                                  int32_t field_index,
+                                  const MapSharedShreddingFieldMeta& expected_meta) const {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(data_file_name));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr file_format,
+                             FileFormatFactory::Get(file_format_str, {}));
+        ASSERT_OK_AND_ASSIGN(auto reader_builder,
+                             file_format->CreateReaderBuilder(/*batch_size=*/10));
+        ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(input_stream));
+        auto c_file_schema = batch_reader->GetFileSchema().value();
+        auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie();
+        ASSERT_TRUE(file_schema->Equals(*expected_schema, /*check_metadata=*/false))
+            << "Expected schema:\n"
+            << expected_schema->ToString() << "\nActual schema:\n"
+            << file_schema->ToString();
+
+        auto metadata = file_schema->field(field_index)->metadata();
+        ASSERT_NE(nullptr, metadata);
+        ASSERT_OK_AND_ASSIGN(auto actual_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
+        ASSERT_EQ(expected_meta, actual_meta);
+    }
+
  private:
     std::shared_ptr pool_;
     std::shared_ptr file_system_;
@@ -145,8 +173,9 @@ TEST_P(PostponeBucketWriterTest, TestSimple) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch
     std::shared_ptr array1 =
@@ -221,9 +250,10 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        std::vector{"key"}, path_factory, /*schema_id=*/1, arrow::schema(fields),
-        options, pool_);
+    ASSERT_OK_AND_ASSIGN(
+        auto postpone_bucket_writer,
+        PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1,
+                                     arrow::schema(fields), options, pool_));
 
     // write batch
     auto array1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
@@ -284,6 +314,75 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) {
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
 }
 
+TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) {
+    const std::string file_format = "parquet";
+    arrow::FieldVector fields = {arrow::field("key", arrow::utf8()),
+                                 arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))};
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap(
+            {{Options::FILE_FORMAT, file_format},
+             {"fields.tags.map.storage-layout", "shared-shredding"},
+             {"fields.tags.map.shared-shredding.max-columns", "3"},
+             {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared();
+    ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
+    std::string uuid = path_factory->uuid_;
+    auto value_schema = arrow::schema(fields);
+    ASSERT_OK_AND_ASSIGN(
+        auto postpone_bucket_writer,
+        PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1,
+                                     value_schema, options, pool_));
+
+    auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        ["Lucy", [["a", 1], ["b", 2]]],
+        ["Bob", [["c", 3], ["a", 4]]]
+    ])")
+                     .ValueOrDie();
+    ASSERT_TRUE(array);
+    WriteBatch(array, /*row_kinds=*/{}, postpone_bucket_writer.get());
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
+                         postpone_bucket_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(postpone_bucket_writer->Close());
+    ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty());
+    ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size());
+
+    std::string data_file_name = "data-" + uuid + "-0." + file_format;
+    std::string data_file_path = dir->Str() + "/" + data_file_name;
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr data_file_status,
+                         options.GetFileSystem()->GetFileStatus(data_file_path));
+    ASSERT_GT(data_file_status->GetLen(), 0);
+
+    arrow::FieldVector write_fields = {arrow::field("_SEQUENCE_NUMBER", arrow::int64()),
+                                       arrow::field("_VALUE_KIND", arrow::int8())};
+    write_fields.insert(write_fields.end(), fields.begin(), fields.end());
+    ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   arrow::schema(write_fields), {{"tags", 3}}));
+
+    MapSharedShreddingFieldMeta expected_meta;
+    expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}};
+    expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}};
+    expected_meta.num_columns = 3;
+    expected_meta.max_row_width = 2;
+
+    CheckShreddingFileSchema(file_format, data_file_path, expected_schema, /*field_index=*/3,
+                             expected_meta);
+
+    auto physical_type = arrow::struct_(expected_schema->fields());
+    std::shared_ptr expected_array;
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([
+        [-1, 0, "Lucy", [[0, 1, -1], 1, 2, null, null]],
+        [-1, 0, "Bob",  [[2, 0, -1], 3, 4, null, null]]
+    ])"},
+                                                                 &expected_array)
+                    .ok());
+    CheckFileContent(file_format, data_file_path, physical_type, expected_array);
+}
+
 TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
     auto file_format = GetParam();
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
@@ -295,8 +394,9 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch 1, batch size = 3
     std::shared_ptr array1 =
@@ -380,6 +480,45 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
 }
 
+TEST_P(PostponeBucketWriterTest, TargetFileRowNumRollsOnlyAfterWholeBatch) {
+    auto file_format = GetParam();
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, file_format},
+                                               {Options::TARGET_FILE_ROW_NUM, "2"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared();
+    ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
+
+    // The first batch exceeds the target by itself. It must remain intact in one file instead of
+    // being sliced at two rows.
+    auto first_batch = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["David", 120, 11, null],
+      ["Bob", 140, 12, null],
+      ["Alex", 110, 10, null]
+    ])")
+                           .ValueOrDie();
+    WriteBatch(first_batch, /*row_kinds=*/{}, postpone_bucket_writer.get());
+
+    auto second_batch = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["Lucy", 20, 1, 14.1]
+    ])")
+                            .ValueOrDie();
+    WriteBatch(second_batch, /*row_kinds=*/{}, postpone_bucket_writer.get());
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         postpone_bucket_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(postpone_bucket_writer->Close());
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+    EXPECT_EQ(3, files[0]->row_count);
+    EXPECT_EQ(1, files[1]->row_count);
+}
+
 TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
     auto file_format = GetParam();
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
@@ -392,8 +531,9 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch 1, batch size = 3
     std::shared_ptr array1 =
@@ -521,8 +661,9 @@ TEST_P(PostponeBucketWriterTest, TestPrepareCommitForEmptyData) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // prepare commit, without write
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
@@ -561,8 +702,9 @@ TEST_P(PostponeBucketWriterTest, TestCloseBeforePrepareCommit) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    auto postpone_bucket_writer = std::make_shared(
-        primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch
     std::shared_ptr array1 =
@@ -593,8 +735,10 @@ TEST_P(PostponeBucketWriterTest, TestIOException) {
         ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
         std::string uuid = path_factory->uuid_;
 
-        auto postpone_bucket_writer = std::make_shared(
-            primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_);
+        ASSERT_OK_AND_ASSIGN(
+            auto postpone_bucket_writer,
+            PostponeBucketWriter::Create(primary_keys_, path_factory,
+                                         /*schema_id=*/1, value_schema_, options, pool_));
 
         // write batch
         std::shared_ptr array =
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp
index 22be2de4..b9ea5da2 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -26,6 +26,8 @@
 #include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
 #include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/variant/variant_access_utils.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/decimal_utils.h"
 #include "paimon/common/utils/string_utils.h"
@@ -56,10 +58,19 @@ Status ArrowSchemaValidator::ValidateSchema(const arrow::Schema& schema) {
 
 Status ArrowSchemaValidator::ValidateSchemaWithFieldId(const arrow::Schema& schema) {
     PAIMON_RETURN_NOT_OK(ValidateSchema(schema));
-    auto struct_type = arrow::struct_(schema.fields());
     std::set field_id_set;
-    PAIMON_RETURN_NOT_OK(
-        ValidateDataTypeWithFieldId(struct_type, /*key_value_metadata=*/nullptr, &field_id_set));
+    for (const auto& field : schema.fields()) {
+        PAIMON_ASSIGN_OR_RAISE(DataField data_field,
+                               DataField::ConvertArrowFieldToDataField(field));
+        auto iter = field_id_set.find(data_field.Id());
+        if (iter != field_id_set.end()) {
+            return Status::Invalid(
+                fmt::format("field id must be unique, duplicate field id {}", data_field.Id()));
+        }
+        field_id_set.insert(data_field.Id());
+        PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(field->type(), field->metadata(),
+                                                         /*allow_blob=*/true, &field_id_set));
+    }
     return Status::OK();
 }
 
@@ -94,7 +105,7 @@ Status ArrowSchemaValidator::ValidateNoWhitespaceOnlyFields(const arrow::FieldVe
 
 Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
     const std::shared_ptr& type,
-    const std::shared_ptr& key_value_metadata,
+    const std::shared_ptr& key_value_metadata, bool allow_blob,
     std::set* field_id_set) {
     const auto kind = type->id();
     switch (kind) {
@@ -115,10 +126,15 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
             const auto& value_field =
                 arrow::internal::checked_cast(type.get())->value_field();
             PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
-                value_field->type(), value_field->metadata(), field_id_set));
+                value_field->type(), value_field->metadata(), /*allow_blob=*/false, field_id_set));
             break;
         }
         case arrow::Type::type::STRUCT: {
+            if (VariantTypeUtils::IsVariantMetadata(key_value_metadata)) {
+                // A variant struct is a leaf type: its value/metadata children carry fixed
+                // paimon field ids 0/1 which must not join the global field id uniqueness check.
+                break;
+            }
             arrow::FieldVector sub_fields =
                 arrow::internal::checked_cast(type.get())->fields();
             for (const auto& sub_field : sub_fields) {
@@ -131,7 +147,7 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
                 }
                 field_id_set->insert(data_field.Id());
                 PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
-                    sub_field->type(), sub_field->metadata(), field_id_set));
+                    sub_field->type(), sub_field->metadata(), /*allow_blob=*/false, field_id_set));
             }
             break;
         }
@@ -140,14 +156,17 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
                 arrow::internal::checked_cast(type.get())->key_field();
             const auto& item_field =
                 arrow::internal::checked_cast(type.get())->item_field();
-            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(key_field->type(),
-                                                             key_field->metadata(), field_id_set));
-            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(item_field->type(),
-                                                             item_field->metadata(), field_id_set));
+            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
+                key_field->type(), key_field->metadata(), /*allow_blob=*/false, field_id_set));
+            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
+                item_field->type(), item_field->metadata(), /*allow_blob=*/false, field_id_set));
             break;
         }
         case arrow::Type::type::LARGE_BINARY: {
             if (BlobUtils::IsBlobMetadata(key_value_metadata)) {
+                if (!allow_blob) {
+                    return Status::Invalid("Blob field must be a top-level field.");
+                }
                 break;
             }
             [[fallthrough]];
@@ -160,6 +179,11 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
 }
 
 Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& field) {
+    return ValidateField(field, /*allow_blob=*/true);
+}
+
+Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& field,
+                                           bool allow_blob) {
     const auto kind = field->type()->id();
     switch (kind) {
         case arrow::Type::type::BOOL:
@@ -181,14 +205,24 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
             const auto& value_field =
                 arrow::internal::checked_cast(*field->type())
                     .value_field();
-            PAIMON_RETURN_NOT_OK(ValidateField(value_field));
+            PAIMON_RETURN_NOT_OK(ValidateField(value_field, /*allow_blob=*/false));
             break;
         }
         case arrow::Type::type::STRUCT: {
+            if (VariantTypeUtils::IsVariantField(field)) {
+                if (VariantAccessUtils::IsVariantAccessType(field->type())) {
+                    // A variant column read as a variant-access projection keeps the variant
+                    // marker but replaces the type with the projection struct; its children are
+                    // cast targets validated by the variant read plans.
+                    break;
+                }
+                PAIMON_RETURN_NOT_OK(VariantTypeUtils::ValidateVariantShape(field));
+                break;
+            }
             arrow::FieldVector arrow_fields =
                 arrow::internal::checked_cast(*field->type()).fields();
             for (const auto& sub_field : arrow_fields) {
-                PAIMON_RETURN_NOT_OK(ValidateField(sub_field));
+                PAIMON_RETURN_NOT_OK(ValidateField(sub_field, /*allow_blob=*/false));
             }
             break;
         }
@@ -197,12 +231,19 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
                 arrow::internal::checked_cast(*field->type()).key_field();
             const auto& item_field =
                 arrow::internal::checked_cast(*field->type()).item_field();
-            PAIMON_RETURN_NOT_OK(ValidateField(key_field));
-            PAIMON_RETURN_NOT_OK(ValidateField(item_field));
+            if (key_field->nullable()) {
+                return Status::Invalid(
+                    fmt::format("Map field '{}' has a nullable key.", field->name()));
+            }
+            PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false));
+            PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false));
             break;
         }
         case arrow::Type::type::LARGE_BINARY: {
             if (BlobUtils::IsBlobField(field)) {
+                if (!allow_blob) {
+                    return Status::Invalid("Blob field must be a top-level field.");
+                }
                 break;
             }
             [[fallthrough]];
diff --git a/src/paimon/core/schema/arrow_schema_validator.h b/src/paimon/core/schema/arrow_schema_validator.h
index 61184458..f7abf6d8 100644
--- a/src/paimon/core/schema/arrow_schema_validator.h
+++ b/src/paimon/core/schema/arrow_schema_validator.h
@@ -58,7 +58,8 @@ class PAIMON_EXPORT ArrowSchemaValidator {
  private:
     static Status ValidateDataTypeWithFieldId(
         const std::shared_ptr& type,
-        const std::shared_ptr& key_value_metadata,
+        const std::shared_ptr& key_value_metadata, bool allow_blob,
         std::set* field_id_set);
+    static Status ValidateField(const std::shared_ptr& field, bool allow_blob);
 };
 }  // namespace paimon
diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp
index f1ec0e5d..0363dff6 100644
--- a/src/paimon/core/schema/arrow_schema_validator_test.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp
@@ -24,6 +24,10 @@
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/variant/variant_access_utils.h"
+#include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/testing/utils/testharness.h"
@@ -151,6 +155,51 @@ TEST(ArrowSchemaValidatorTest, TestInvalidDataType) {
     }
 }
 
+TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) {
+    {
+        auto arrow_schema =
+            arrow::schema(arrow::FieldVector({BlobUtils::ToArrowField("blob", true)}));
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema));
+    }
+    {
+        std::vector fields = {DataField(0, BlobUtils::ToArrowField("blob", true))};
+        auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema));
+    }
+    {
+        auto nested_blob_field =
+            arrow::field("nested", arrow::struct_({BlobUtils::ToArrowField("blob", true)}));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({nested_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        auto array_blob_field =
+            arrow::field("array_blob", arrow::list(BlobUtils::ToArrowField("item", true)));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({array_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        auto map_blob_field = arrow::field(
+            "map_blob",
+            arrow::map(arrow::utf8(), arrow::struct_({BlobUtils::ToArrowField("blob", true)})));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({map_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        std::vector nested_fields = {
+            DataField(1, BlobUtils::ToArrowField("blob", true))};
+        std::vector fields = {DataField(
+            0,
+            arrow::field("nested", DataField::ConvertDataFieldsToArrowStructType(nested_fields)))};
+        auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+}
+
 TEST(ArrowSchemaValidatorTest, ValidateDataTypeWithFieldId) {
     {
         std::vector fields = {DataField(3, arrow::field("f3", arrow::float64())),
@@ -268,11 +317,53 @@ TEST(ArrowSchemaValidatorTest, ValidateDataTypeWithFieldId) {
         auto struct_type = DataField::ConvertDataFieldsToArrowStructType(fields);
         std::set field_id_set;
         ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateDataTypeWithFieldId(
-                                struct_type, /*key_value_metadata=*/nullptr, &field_id_set),
+                                struct_type, /*key_value_metadata=*/nullptr,
+                                /*allow_blob=*/true, &field_id_set),
                             "Unknown or unsupported arrow type: large_string");
     }
 }
 
+TEST(ArrowSchemaValidatorTest, ValidateVariantField) {
+    // A variant field validates as a leaf: its fixed child ids 0/1 do not join the global field
+    // id uniqueness check, even when top-level fields use ids 0/1.
+    {
+        std::vector fields = {DataField(0, arrow::field("f0", arrow::utf8())),
+                                         DataField(1, VariantTypeUtils::ToArrowField("v1")),
+                                         DataField(2, VariantTypeUtils::ToArrowField("v2"))};
+        auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema))
+            << ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema).ToString();
+    }
+    // A variant-access projection (variant marker + description-carrying children) validates
+    // as a leaf instead of being shape-checked.
+    {
+        auto age_child =
+            arrow::field("0", arrow::int64(), true,
+                         arrow::KeyValueMetadata::Make(
+                             {DataField::DESCRIPTION},
+                             {VariantAccessUtils::BuildVariantMetadata("$.age", true, "UTC")}));
+        std::unordered_map metadata = {
+            {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}};
+        auto access_field = arrow::field("v", arrow::struct_({age_child}), true,
+                                         std::make_shared(metadata));
+        auto arrow_schema = arrow::schema({access_field});
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema))
+            << ArrowSchemaValidator::ValidateSchema(*arrow_schema).ToString();
+    }
+    // A variant-marked struct with the wrong physical shape is rejected.
+    {
+        std::unordered_map metadata = {
+            {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}};
+        auto bad_variant =
+            arrow::field("v",
+                         arrow::struct_({arrow::field("value", arrow::binary(), true),
+                                         arrow::field("metadata", arrow::binary(), true)}),
+                         true, std::make_shared(metadata));
+        auto arrow_schema = arrow::schema({bad_variant});
+        ASSERT_NOK(ArrowSchemaValidator::ValidateSchema(*arrow_schema));
+    }
+}
+
 TEST(ArrowSchemaValidatorTest, ContainTimestampWithTimezone) {
     auto timezone = DateTimeUtils::GetLocalTimezoneName();
     {
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index 17037ed9..ed79e566 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -21,6 +21,7 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -30,9 +31,12 @@
 #include 
 
 #include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
 #include "fmt/ranges.h"
 #include "paimon/common/data/blob_utils.h"
+#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/object_utils.h"
@@ -41,6 +45,7 @@
 #include "paimon/core/core_options.h"
 #include "paimon/core/options/changelog_producer.h"
 #include "paimon/core/options/expire_config.h"
+#include "paimon/core/options/map_storage_layout.h"
 #include "paimon/core/options/merge_engine.h"
 #include "paimon/core/schema/arrow_schema_validator.h"
 #include "paimon/core/schema/table_schema.h"
@@ -49,10 +54,74 @@
 #include "paimon/result.h"
 
 namespace paimon {
+namespace {
+
+bool ContainsBlobField(const std::shared_ptr& field) {
+    if (BlobUtils::IsBlobField(field)) {
+        return true;
+    }
+    const std::shared_ptr& type = field->type();
+    if (type->id() == arrow::Type::STRUCT) {
+        for (const auto& child : type->fields()) {
+            if (ContainsBlobField(child)) {
+                return true;
+            }
+        }
+    } else if (type->id() == arrow::Type::LIST) {
+        return ContainsBlobField(type->fields().front());
+    } else if (type->id() == arrow::Type::MAP) {
+        const auto& map_type = arrow::internal::checked_cast(*type);
+        return ContainsBlobField(map_type.key_field()) || ContainsBlobField(map_type.item_field());
+    }
+    return false;
+}
+
+Status ValidateSharedShreddingCompression(const std::string& option_key,
+                                          const std::string& compression) {
+    std::string normalized = StringUtils::ToLowerCase(compression);
+    if (normalized != "none" && normalized != "lz4" && normalized != "zstd") {
+        return Status::Invalid(fmt::format(
+            "MAP shared-shredding only supports none/lz4/zstd compression, but {} is {}.",
+            option_key, compression));
+    }
+    return Status::OK();
+}
+
+Status ValidateSharedShreddingFileFormat(const std::string& option_key,
+                                         const std::string& file_format) {
+    std::string normalized = StringUtils::ToLowerCase(file_format);
+    if (normalized != "parquet" && normalized != "orc") {
+        return Status::Invalid(fmt::format(
+            "MAP shared-shredding only supports parquet/orc file formats, but {} is {}.",
+            option_key, file_format));
+    }
+    return Status::OK();
+}
+
+Status ValidatePerLevelOption(
+    const std::map& options, const std::string& option_key,
+    const std::function& validator) {
+    auto it = options.find(option_key);
+    if (it == options.end() || it->second.empty()) {
+        return Status::OK();
+    }
+    auto entries = StringUtils::Split(it->second, std::string(","));
+    for (const std::string& entry : entries) {
+        auto level_and_value = StringUtils::Split(entry, std::string(":"));
+        if (level_and_value.size() == 2) {
+            PAIMON_RETURN_NOT_OK(
+                validator(option_key + "." + level_and_value[0], level_and_value[1]));
+        }
+    }
+    return Status::OK();
+}
+
+}  // namespace
 
 bool SchemaValidation::IsComplexType(const std::shared_ptr& field) {
-    return (field->type()->id() == arrow::Type::TIMESTAMP ||
-            field->type()->id() == arrow::Type::DECIMAL || BlobUtils::IsBlobField(field));
+    arrow::Type::type arrow_type_id = field->type()->id();
+    return (arrow_type_id == arrow::Type::TIMESTAMP || arrow_type_id == arrow::Type::DECIMAL128 ||
+            BlobUtils::IsBlobField(field));
 }
 
 Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
@@ -104,14 +173,10 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
     // TODO(yonghao.fyh): check changelog num retain
     // TODO(yonghao.fyh): support file format validate data fields
     for (const auto& field_name : field_names) {
-        if (SpecialFields::IsSpecialFieldName(field_name)) {
+        if (SpecialFields::IsSystemField(field_name)) {
             return Status::Invalid(
                 fmt::format("field name '{}' in schema cannot be special field.", field_name));
         }
-        if (StringUtils::StartsWith(field_name, SpecialFields::KEY_FIELD_PREFIX)) {
-            return Status::Invalid(fmt::format("field name '{}' in schema cannot start with '{}'.",
-                                               field_name, SpecialFields::KEY_FIELD_PREFIX));
-        }
     }
     // TODO(yonghao.fyh): check streaming read overwrite
     // TODO(yonghao.fyh): check 'partition.expiration-time'
@@ -122,6 +187,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
 
     PAIMON_RETURN_NOT_OK(ValidateRowTracking(schema, options));
     PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options));
+    PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options));
     return Status::OK();
 }
 
@@ -449,13 +515,7 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor
     const auto& configured_blob_names = options.GetBlobFields();
     const auto& blob_descriptor_names = options.GetBlobDescriptorFields();
     const auto& blob_view_names = options.GetBlobViewFields();
-    const auto& blob_external_storage_names = options.GetBlobExternalStorageFields();
-    std::vector configured_blob_like_names = configured_blob_names;
-    configured_blob_like_names.insert(configured_blob_like_names.end(),
-                                      blob_descriptor_names.begin(), blob_descriptor_names.end());
-    configured_blob_like_names.insert(configured_blob_like_names.end(), blob_view_names.begin(),
-                                      blob_view_names.end());
-    if (configured_blob_like_names.empty() && blob_external_storage_names.empty()) {
+    if (configured_blob_names.empty() && blob_descriptor_names.empty() && blob_view_names.empty()) {
         return Status::OK();
     }
 
@@ -480,8 +540,6 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor
     PAIMON_RETURN_NOT_OK(
         validate_blob_fields(blob_descriptor_names, Options::BLOB_DESCRIPTOR_FIELD));
     PAIMON_RETURN_NOT_OK(validate_blob_fields(blob_view_names, Options::BLOB_VIEW_FIELD));
-    PAIMON_RETURN_NOT_OK(
-        validate_blob_fields(blob_external_storage_names, Options::BLOB_EXTERNAL_STORAGE_FIELD));
 
     std::set blob_descriptor_name_set(blob_descriptor_names.begin(),
                                                    blob_descriptor_names.end());
@@ -492,22 +550,104 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor
                                                Options::BLOB_DESCRIPTOR_FIELD));
         }
     }
+    return Status::OK();
+}
+
+Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
+                                                  const CoreOptions& options) {
+    // Extract all field names that have map.storage-layout configured from options
+    const std::string layout_suffix = std::string(".") + std::string(Options::MAP_STORAGE_LAYOUT);
+    const auto& options_map = options.ToMap();
+
+    std::unordered_map> schema_fields;
+    for (const auto& field : schema.Fields()) {
+        schema_fields[field.Name()] = field.Type();
+    }
 
-    for (const auto& blob_external_storage_name : blob_external_storage_names) {
-        if (blob_descriptor_name_set.count(blob_external_storage_name) == 0) {
+    std::string fields_prefix_str = std::string(Options::FIELDS_PREFIX);
+    bool has_shared_shredding = false;
+    for (const auto& [key, value] : options_map) {
+        if (!StringUtils::StartsWith(key, fields_prefix_str)) {
+            continue;
+        }
+        if (!StringUtils::EndsWith(key, layout_suffix)) {
+            continue;
+        }
+        // key = "fields..map.storage-layout"
+        // Extract field_name: skip "fields." prefix and ".map.storage-layout" suffix
+        std::string field_name =
+            key.substr(fields_prefix_str.size() + 1,
+                       key.size() - fields_prefix_str.size() - 1 - layout_suffix.size());
+
+        // Check field exists in schema
+        auto it = schema_fields.find(field_name);
+        if (it == schema_fields.end()) {
             return Status::Invalid(
-                fmt::format("Field '{}' in '{}' must also be in '{}'.", blob_external_storage_name,
-                            Options::BLOB_EXTERNAL_STORAGE_FIELD, Options::BLOB_DESCRIPTOR_FIELD));
+                fmt::format("Column '{}' is configured with map.storage-layout "
+                            "but does not exist in table schema.",
+                            field_name));
         }
-    }
-    if (!blob_external_storage_names.empty()) {
-        auto external_storage_path = options.GetBlobExternalStoragePath();
-        if (!external_storage_path || external_storage_path->empty()) {
-            return Status::Invalid(fmt::format("'{}' must be set when '{}' is configured.",
-                                               Options::BLOB_EXTERNAL_STORAGE_PATH,
-                                               Options::BLOB_EXTERNAL_STORAGE_FIELD));
+
+        // Any column configured with map.storage-layout must be a MAP type
+        const auto& field_type = it->second;
+        if (field_type->id() != arrow::Type::MAP) {
+            return Status::Invalid(
+                fmt::format("Column '{}' is configured with map.storage-layout "
+                            "but its type is not MAP.",
+                            field_name));
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(MapStorageLayout layout, options.GetMapStorageLayout(field_name));
+        if (layout != MapStorageLayout::SHARED_SHREDDING) {
+            continue;
+        }
+        has_shared_shredding = true;
+        for (const auto& field : schema.Fields()) {
+            if (VariantTypeUtils::ContainsVariantField(field.ArrowField())) {
+                return Status::Invalid(
+                    "MAP shared-shredding currently cannot be used with Variant fields.");
+            }
+        }
+        // Column configured with shared-shredding must be MAP
+        if (!MapSharedShreddingUtils::IsShreddingKeyMap(field_type)) {
+            return Status::Invalid(
+                fmt::format("Column '{}' is configured with map.storage-layout=shared-shredding "
+                            "but its type is not MAP.",
+                            field_name));
         }
+        auto map_type = arrow::internal::checked_pointer_cast(field_type);
+        if (map_type->key_field()->nullable()) {
+            return Status::Invalid(
+                fmt::format("Column '{}' is configured with map.storage-layout=shared-shredding "
+                            "but its map key type is nullable.",
+                            field_name));
+        }
+        if (ContainsBlobField(map_type->item_field())) {
+            return Status::Invalid("MAP shared-shredding currently cannot contain BLOB fields.");
+        }
+        // Validate max-columns config
+        PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name));
+        // Validate placement policy config
+        PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingColumnPlacementPolicy(field_name));
     }
+    if (!has_shared_shredding) {
+        return Status::OK();
+    }
+
+    if (IsPostponeBucketTable(schema, options.GetBucket())) {
+        return Status::Invalid(
+            "MAP shared-shredding currently does not support postpone bucket mode.");
+    }
+
+    PAIMON_RETURN_NOT_OK(ValidateSharedShreddingFileFormat(Options::FILE_FORMAT,
+                                                           options.GetFileFormat()->Identifier()));
+    PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_FORMAT_PER_LEVEL,
+                                                ValidateSharedShreddingFileFormat));
+    PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::FILE_COMPRESSION,
+                                                            options.GetFileCompression()));
+    PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_COMPRESSION_PER_LEVEL,
+                                                ValidateSharedShreddingCompression));
+
     return Status::OK();
 }
 
diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h
index 778f8a35..613372ff 100644
--- a/src/paimon/core/schema/schema_validation.h
+++ b/src/paimon/core/schema/schema_validation.h
@@ -73,6 +73,8 @@ class SchemaValidation {
 
     static Status ValidateBlobFields(const TableSchema& schema, const CoreOptions& options);
 
+    static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options);
+
     static bool IsComplexType(const std::shared_ptr& field);
 };
 
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 9dc7472a..57179e8c 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -104,14 +104,11 @@ TEST(SchemaValidationTest, TestWithBlobField) {
         auto schema = arrow::schema(fields);
         std::vector primary_keys = {};
         std::vector partition_keys = {"f1"};
-        std::map options = {
-            {Options::BUCKET, "-1"},
-            {Options::ROW_TRACKING_ENABLED, "true"},
-            {Options::DATA_EVOLUTION_ENABLED, "true"},
-            {Options::BLOB_DESCRIPTOR_FIELD, "f3"},
-            {Options::BLOB_VIEW_FIELD, "f4"},
-            {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f3"},
-            {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}};
+        std::map options = {{Options::BUCKET, "-1"},
+                                                      {Options::ROW_TRACKING_ENABLED, "true"},
+                                                      {Options::DATA_EVOLUTION_ENABLED, "true"},
+                                                      {Options::BLOB_DESCRIPTOR_FIELD, "f3"},
+                                                      {Options::BLOB_VIEW_FIELD, "f4"}};
         ASSERT_OK_AND_ASSIGN(
             std::shared_ptr table_schema,
             TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
@@ -150,42 +147,6 @@ TEST(SchemaValidationTest, TestWithBlobField) {
             SchemaValidation::ValidateTableSchema(*table_schema),
             "Field 'f3' in 'blob-view-field' can not also be in 'blob-descriptor-field'.");
     }
-    {
-        arrow::FieldVector fields = {f0, f1, f2, f3, f4};
-        auto schema = arrow::schema(fields);
-        std::vector primary_keys = {};
-        std::vector partition_keys = {"f1"};
-        std::map options = {
-            {Options::BUCKET, "-1"},
-            {Options::ROW_TRACKING_ENABLED, "true"},
-            {Options::DATA_EVOLUTION_ENABLED, "true"},
-            {Options::BLOB_DESCRIPTOR_FIELD, "f3"},
-            {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f4"},
-            {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}};
-        ASSERT_OK_AND_ASSIGN(
-            std::shared_ptr table_schema,
-            TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
-        ASSERT_NOK_WITH_MSG(
-            SchemaValidation::ValidateTableSchema(*table_schema),
-            "Field 'f4' in 'blob-external-storage-field' must also be in 'blob-descriptor-field'.");
-    }
-    {
-        arrow::FieldVector fields = {f0, f1, f2, f3};
-        auto schema = arrow::schema(fields);
-        std::vector primary_keys = {};
-        std::vector partition_keys = {"f1"};
-        std::map options = {{Options::BUCKET, "-1"},
-                                                      {Options::ROW_TRACKING_ENABLED, "true"},
-                                                      {Options::DATA_EVOLUTION_ENABLED, "true"},
-                                                      {Options::BLOB_DESCRIPTOR_FIELD, "f3"},
-                                                      {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f3"}};
-        ASSERT_OK_AND_ASSIGN(
-            std::shared_ptr table_schema,
-            TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
-        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "'blob-external-storage-path' must be set when "
-                            "'blob-external-storage-field' is configured.");
-    }
     {
         arrow::FieldVector fields = {f0, f1, f2, f3};
         auto schema = arrow::schema(fields);
@@ -764,7 +725,7 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) {
             TableSchema::Create(/*schema_id=*/0, invalid_schema, /*partition_keys=*/{},
                                 /*primary_keys=*/{}, /*options=*/{}));
         ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "field name '_KEY_a' in schema cannot start with '_KEY_'");
+                            "field name '_KEY_a' in schema cannot be special field.");
     }
     {
         std::map options = {{Options::CHANGELOG_PRODUCER, "input"},
@@ -823,4 +784,317 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) {
                             "Data evolution config must disabled with deletion-vectors.enabled");
     }
 }
+
+TEST(SchemaValidationTest, TestMapStorageLayout) {
+    auto f0 = arrow::field("f0", arrow::utf8());
+    auto f1 = arrow::field("f1", arrow::int32());
+    auto f2 = arrow::field("f2", arrow::map(arrow::utf8(), arrow::int64()));
+    auto f3 = arrow::field("f3", arrow::map(arrow::int32(), arrow::utf8()));
+
+    // Valid: shared-shredding on MAP column
+    {
+        arrow::FieldVector fields = {f0, f1, f2};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.f2.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    // Invalid: field not in schema failed in ValidateFieldsPrefix
+    {
+        arrow::FieldVector fields = {f0, f1};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.nonexist.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "Field nonexist can not be found in table schema");
+    }
+    // Invalid: field not in schema failed in ValidateMapStorageLayout
+    {
+        arrow::FieldVector fields = {f0, f1};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.nonexist.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateMapStorageLayout(*table_schema, core_options),
+                            "Column 'nonexist' is configured with map.storage-layout but does not "
+                            "exist in table schema.");
+    }
+
+    // Invalid: non-MAP column configured with map.storage-layout (any value)
+    {
+        arrow::FieldVector fields = {f0, f1};
+        auto schema = arrow::schema(fields);
+        std::map options = {{Options::BUCKET, "2"},
+                                                      {Options::BUCKET_KEY, "f0"},
+                                                      {"fields.f1.map.storage-layout", "default"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "not MAP");
+    }
+    // Invalid: shared-shredding on non-MAP column
+    {
+        arrow::FieldVector fields = {f0, f1};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.f1.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "not MAP");
+    }
+    // Invalid: shared-shredding on MAP with non-STRING key
+    {
+        arrow::FieldVector fields = {f0, f1, f3};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.f3.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "not MAP");
+    }
+    // Invalid: nested MAP paths are not shared-shredding columns; only top-level columns are
+    // addressable by fields..map.storage-layout.
+    {
+        auto payload = arrow::field(
+            "payload",
+            arrow::struct_({arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64()))}));
+        arrow::FieldVector fields = {f0, f1, payload};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.payload.attrs.map.storage-layout", "shared-shredding"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(
+            SchemaValidation::ValidateTableSchema(*table_schema),
+            "Column 'payload.attrs' is configured with map.storage-layout but does not exist in "
+            "table schema.");
+    }
+    // Valid: default layout on a MAP column
+    {
+        arrow::FieldVector fields = {f0, f1, f2};
+        auto schema = arrow::schema(fields);
+        std::map options = {{Options::BUCKET, "2"},
+                                                      {Options::BUCKET_KEY, "f0"},
+                                                      {"fields.f2.map.storage-layout", "default"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    // Invalid: shared-shredding with invalid max-columns
+    {
+        arrow::FieldVector fields = {f0, f1, f2};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.f2.map.storage-layout", "shared-shredding"},
+            {"fields.f2.map.shared-shredding.max-columns", "0"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "options map.shared-shredding.max-columns must > 0");
+    }
+    // Invalid: shared-shredding with invalid placement policy
+    {
+        arrow::FieldVector fields = {f0, f1, f2};
+        auto schema = arrow::schema(fields);
+        std::map options = {
+            {Options::BUCKET, "2"},
+            {Options::BUCKET_KEY, "f0"},
+            {"fields.f2.map.storage-layout", "shared-shredding"},
+            {"fields.f2.map.shared-shredding.column-placement-policy", "invalid"}};
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{"f0", "f1"}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "invalid map.shared-shredding.column-placement-policy: invalid");
+    }
+}
+
+TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) {
+    auto nullable_key_map =
+        std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/true),
+                                         arrow::field("value", arrow::int64()));
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", nullable_key_map),
+    });
+    std::map options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+    ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                            /*primary_keys=*/{}, options),
+                        "Map field 'f1' has a nullable key.");
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) {
+    auto direct_blob_map =
+        arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true));
+    auto nested_blob_map = arrow::map(
+        arrow::utf8(), arrow::field("value", arrow::struct_({BlobUtils::ToArrowField("blob")})));
+    std::map options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const auto& map_type : {direct_blob_map, nested_blob_map}) {
+        auto schema = arrow::schema({
+            arrow::field("f0", arrow::utf8()),
+            arrow::field("f1", map_type),
+        });
+        ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                /*primary_keys=*/{}, options),
+                            "Blob field must be a top-level field.");
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingCompression) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map base_options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const std::string compression : {"none", "lz4", "zstd", "ZSTD"}) {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = compression;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = "snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression is snappy.");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = "";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression is .");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION_PER_LEVEL] = "0:lz4,1:snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression.per.level.1 is snappy.");
+    }
+    {
+        auto options = base_options;
+        options.erase("fields.f1.map.storage-layout");
+        options[Options::FILE_COMPRESSION] = "snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingFileFormat) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map base_options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const std::string file_format : {"parquet", "orc"}) {
+        auto options = base_options;
+        options[Options::FILE_FORMAT] = file_format;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_FORMAT] = "avro";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(
+            SchemaValidation::ValidateTableSchema(*table_schema),
+            "MAP shared-shredding only supports parquet/orc file formats, but file.format is "
+            "avro.");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_FORMAT_PER_LEVEL] = "0:parquet,1:avro";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports parquet/orc file formats, but "
+                            "file.format.per.level.1 is avro.");
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map options = {
+        {Options::BUCKET, "-2"},
+        {Options::WRITE_ONLY, "true"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{"f0"}, options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "MAP shared-shredding currently does not support postpone bucket mode.");
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp
index a0294617..2beff595 100644
--- a/src/paimon/core/schema/table_schema.cpp
+++ b/src/paimon/core/schema/table_schema.cpp
@@ -28,6 +28,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
@@ -48,9 +49,6 @@ Result> TableSchema::Create(
     int64_t schema_id, const std::shared_ptr& schema,
     const std::vector& partition_keys, const std::vector& primary_keys,
     const std::map& options) {
-    if (schema_id != 0) {
-        return Status::NotImplemented("do not support schema evolution, schema_id must be 0");
-    }
     std::vector data_fields;
     int32_t field_id = 0;
     std::set primary_key_set;
@@ -99,6 +97,11 @@ Result> TableSchema::AssignFieldIdsRecursively(
     }
     auto type = field->type();
     if (type->id() == arrow::Type::STRUCT) {
+        if (VariantTypeUtils::IsVariantField(field)) {
+            // A variant struct is a leaf type: its value/metadata children keep their fixed
+            // paimon field ids 0/1 (mapped to parquet field ids on write).
+            return metadata ? field->WithMergedMetadata(metadata) : field;
+        }
         auto struct_type = arrow::internal::checked_pointer_cast(field->type());
         arrow::FieldVector new_childs;
         for (const auto& child : struct_type->fields()) {
@@ -122,8 +125,11 @@ Result> TableSchema::AssignFieldIdsRecursively(
             key_field, AssignFieldIdsRecursively(key_field, /*set_field_id=*/false, field_id));
         PAIMON_ASSIGN_OR_RAISE(
             value_field, AssignFieldIdsRecursively(value_field, /*set_field_id=*/false, field_id));
-        return arrow::field(field->name(), arrow::map(key_field->type(), value_field),
-                            field->nullable(), metadata);
+        // Paimon MAP does not expose Arrow's keys_sorted property. Normalize it so an
+        // in-memory schema and the same schema reloaded from JSON remain equivalent.
+        auto new_map_type =
+            std::make_shared(key_field, value_field, /*keys_sorted=*/false);
+        return arrow::field(field->name(), new_map_type, field->nullable(), metadata);
     }
     return metadata ? field->WithMergedMetadata(metadata) : field;
 }
@@ -193,7 +199,7 @@ std::vector TableSchema::FieldNames() const {
 
 Result TableSchema::GetFieldType(const std::string& field_name) const {
     PAIMON_ASSIGN_OR_RAISE(DataField field, GetField(field_name));
-    return FieldTypeUtils::ConvertToFieldType(field.Type()->id());
+    return FieldTypeUtils::ConvertToFieldType(field.ArrowField());
 }
 
 Result TableSchema::GetField(const std::string& field_name) const {
diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h
index c878151a..bc0c94e4 100644
--- a/src/paimon/core/schema/table_schema.h
+++ b/src/paimon/core/schema/table_schema.h
@@ -113,6 +113,10 @@ class TableSchema : public DataSchema, public Jsonizable {
 
     bool CrossPartitionUpdate() const;
 
+    int64_t TimeMillis() const {
+        return time_millis_;
+    }
+
  private:
     JSONIZABLE_FRIEND_AND_DEFAULT_CTOR(TableSchema);
 
diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp
index dedb3bc9..988f17f9 100644
--- a/src/paimon/core/schema/table_schema_test.cpp
+++ b/src/paimon/core/schema/table_schema_test.cpp
@@ -24,6 +24,7 @@
 #include "arrow/api.h"
 #include "arrow/util/checked_cast.h"
 #include "gtest/gtest.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/fs/local/local_file_system.h"
@@ -136,6 +137,20 @@ TEST_F(TableSchemaTest, TestCreateWithAllFieldsHaveFieldId) {
     ASSERT_EQ(table_schema->Options(), options);
 }
 
+TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) {
+    arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+                                 VariantTypeUtils::ToArrowField("v")};
+    ASSERT_OK_AND_ASSIGN(auto table_schema,
+                         TableSchema::Create(/*schema_id=*/0, arrow::schema(fields),
+                                             /*partition_keys=*/{}, /*primary_keys=*/{},
+                                             /*options=*/{}));
+    ASSERT_OK_AND_ASSIGN(FieldType id_type, table_schema->GetFieldType("id"));
+    ASSERT_EQ(id_type, FieldType::INT);
+    // The variant marker must disambiguate the physical struct into FieldType::VARIANT.
+    ASSERT_OK_AND_ASSIGN(FieldType variant_type, table_schema->GetFieldType("v"));
+    ASSERT_EQ(variant_type, FieldType::VARIANT);
+}
+
 TEST_F(TableSchemaTest, TestInvalidCreate) {
     // partial fields have field id
     arrow::FieldVector fields = MakeArrowField(3);
@@ -1264,6 +1279,34 @@ TEST_F(TableSchemaTest, MapKeyMustBeNotNull) {
     })";
     ASSERT_NOK_WITH_MSG(TableSchema::CreateFromJson(table_schema_str),
                         "Map field 'f0' has a nullable key.");
+
+    auto nullable_key_map =
+        std::make_shared(arrow::field("key", arrow::int8(), /*nullable=*/true),
+                                         arrow::field("value", arrow::int16()));
+    ASSERT_NOK_WITH_MSG(
+        TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", nullable_key_map)}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}),
+        "Map field 'f0' has a nullable key.");
+}
+
+TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) {
+    auto sorted_map =
+        std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/false),
+                                         arrow::field("value", arrow::int64()),
+                                         /*keys_sorted=*/true);
+    ASSERT_OK_AND_ASSIGN(
+        auto table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", sorted_map)}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}));
+
+    auto map_type = std::static_pointer_cast(table_schema->Fields()[0].Type());
+    ASSERT_FALSE(map_type->keys_sorted());
+
+    ASSERT_OK_AND_ASSIGN(std::string json, table_schema->ToJsonString());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr restored, TableSchema::CreateFromJson(json));
+    auto restored_map_type = std::static_pointer_cast(restored->Fields()[0].Type());
+    ASSERT_FALSE(restored_map_type->keys_sorted());
+    ASSERT_TRUE(map_type->Equals(*restored_map_type));
 }
 
 TEST_F(TableSchemaTest, CrossPartitionUpdate) {
diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp
index 5ba177ea..d3081c91 100644
--- a/src/paimon/core/snapshot.cpp
+++ b/src/paimon/core/snapshot.cpp
@@ -81,7 +81,7 @@ bool Snapshot::TEST_Equal(const Snapshot& other) const {
     return version_ == other.version_ && id_ == other.id_ && schema_id_ == other.schema_id_ &&
            index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ &&
            commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ &&
-           log_offsets_ == other.log_offsets_ && total_record_count_ == other.total_record_count_ &&
+           total_record_count_ == other.total_record_count_ &&
            delta_record_count_ == other.delta_record_count_ &&
            changelog_record_count_ == other.changelog_record_count_ &&
            watermark_ == other.watermark_ && statistics_ == other.statistics_ &&
@@ -101,8 +101,7 @@ bool Snapshot::operator==(const Snapshot& other) const {
            changelog_manifest_list_size_ == other.changelog_manifest_list_size_ &&
            index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ &&
            commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ &&
-           time_millis_ == other.time_millis_ && log_offsets_ == other.log_offsets_ &&
-           total_record_count_ == other.total_record_count_ &&
+           time_millis_ == other.time_millis_ && total_record_count_ == other.total_record_count_ &&
            delta_record_count_ == other.delta_record_count_ &&
            changelog_record_count_ == other.changelog_record_count_ &&
            watermark_ == other.watermark_ && statistics_ == other.statistics_ &&
@@ -147,9 +146,7 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc
                    const std::optional& changelog_manifest_list_size,
                    const std::optional& index_manifest, const std::string& commit_user,
                    int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis,
-                   const std::optional>& log_offsets,
-                   const std::optional& total_record_count,
-                   const std::optional& delta_record_count,
+                   int64_t total_record_count, int64_t delta_record_count,
                    const std::optional& changelog_record_count,
                    const std::optional& watermark,
                    const std::optional& statistics,
@@ -169,7 +166,6 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc
       commit_identifier_(commit_identifier),
       commit_kind_(commit_kind),
       time_millis_(time_millis),
-      log_offsets_(log_offsets),
       total_record_count_(total_record_count),
       delta_record_count_(delta_record_count),
       changelog_record_count_(changelog_record_count),
@@ -203,9 +199,11 @@ rapidjson::Value Snapshot::ToJson(rapidjson::Document::AllocatorType* allocator)
                       RapidJsonUtil::SerializeValue(delta_manifest_list_size_, allocator).Move(),
                       *allocator);
     }
-    obj.AddMember(rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST),
-                  RapidJsonUtil::SerializeValue(changelog_manifest_list_, allocator).Move(),
-                  *allocator);
+    if (changelog_manifest_list_) {
+        obj.AddMember(rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST),
+                      RapidJsonUtil::SerializeValue(changelog_manifest_list_, allocator).Move(),
+                      *allocator);
+    }
     if (changelog_manifest_list_size_) {
         obj.AddMember(
             rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST_SIZE),
@@ -230,17 +228,10 @@ rapidjson::Value Snapshot::ToJson(rapidjson::Document::AllocatorType* allocator)
 
     obj.AddMember(rapidjson::StringRef(FIELD_TIME_MILLIS),
                   RapidJsonUtil::SerializeValue(time_millis_, allocator).Move(), *allocator);
-    if (log_offsets_ != std::nullopt) {
-        obj.AddMember(rapidjson::StringRef(FIELD_LOG_OFFSETS),
-                      RapidJsonUtil::SerializeValue(log_offsets_.value(), allocator).Move(),
-                      *allocator);
-    }
     obj.AddMember(rapidjson::StringRef(FIELD_TOTAL_RECORD_COUNT),
-                  RapidJsonUtil::SerializeValue(total_record_count_.value(), allocator).Move(),
-                  *allocator);
+                  RapidJsonUtil::SerializeValue(total_record_count_, allocator).Move(), *allocator);
     obj.AddMember(rapidjson::StringRef(FIELD_DELTA_RECORD_COUNT),
-                  RapidJsonUtil::SerializeValue(delta_record_count_.value(), allocator).Move(),
-                  *allocator);
+                  RapidJsonUtil::SerializeValue(delta_record_count_, allocator).Move(), *allocator);
 
     if (changelog_record_count_ != std::nullopt) {
         obj.AddMember(
@@ -300,12 +291,10 @@ void Snapshot::FromJson(const rapidjson::Value& obj) noexcept(false) {
         throw std::invalid_argument("deserialize CommitKind failed");
     }
     time_millis_ = RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TIME_MILLIS);
-    log_offsets_ = RapidJsonUtil::DeserializeKeyValue>>(
-        obj, FIELD_LOG_OFFSETS);
     total_record_count_ =
-        RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_TOTAL_RECORD_COUNT);
+        RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TOTAL_RECORD_COUNT);
     delta_record_count_ =
-        RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_DELTA_RECORD_COUNT);
+        RapidJsonUtil::DeserializeKeyValue(obj, FIELD_DELTA_RECORD_COUNT);
     changelog_record_count_ = RapidJsonUtil::DeserializeKeyValue>(
         obj, FIELD_CHANGELOG_RECORD_COUNT);
     watermark_ = RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_WATERMARK);
diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h
index 827d416e..1b7f6593 100644
--- a/src/paimon/core/snapshot.h
+++ b/src/paimon/core/snapshot.h
@@ -58,6 +58,11 @@ class Snapshot : public Jsonizable {
         bool operator==(const CommitKind& other) const {
             return value_ == other.value_;
         }
+
+        bool operator!=(const CommitKind& other) const {
+            return value_ != other.value_;
+        }
+
         static std::string ToString(const CommitKind& kind);
         static CommitKind FromString(const std::string& kind);
 
@@ -79,7 +84,6 @@ class Snapshot : public Jsonizable {
     static constexpr char FIELD_COMMIT_IDENTIFIER[] = "commitIdentifier";
     static constexpr char FIELD_COMMIT_KIND[] = "commitKind";
     static constexpr char FIELD_TIME_MILLIS[] = "timeMillis";
-    static constexpr char FIELD_LOG_OFFSETS[] = "logOffsets";
     static constexpr char FIELD_TOTAL_RECORD_COUNT[] = "totalRecordCount";
     static constexpr char FIELD_DELTA_RECORD_COUNT[] = "deltaRecordCount";
     static constexpr char FIELD_CHANGELOG_RECORD_COUNT[] = "changelogRecordCount";
@@ -98,9 +102,7 @@ class Snapshot : public Jsonizable {
              const std::optional& changelog_manifest_list_size,
              const std::optional& index_manifest, const std::string& commit_user,
              int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis,
-             const std::optional>& log_offsets,
-             const std::optional& total_record_count,
-             const std::optional& delta_record_count,
+             int64_t total_record_count, int64_t delta_record_count,
              const std::optional& changelog_record_count,
              const std::optional& watermark, const std::optional& statistics,
              const std::optional>& properties,
@@ -108,7 +110,7 @@ class Snapshot : public Jsonizable {
         : Snapshot(CURRENT_VERSION, id, schema_id, base_manifest_list, base_manifest_list_size,
                    delta_manifest_list, delta_manifest_list_size, changelog_manifest_list,
                    changelog_manifest_list_size, index_manifest, commit_user, commit_identifier,
-                   commit_kind, time_millis, log_offsets, total_record_count, delta_record_count,
+                   commit_kind, time_millis, total_record_count, delta_record_count,
                    changelog_record_count, watermark, statistics, properties, next_row_id) {}
 
     Snapshot(const std::optional& version, int64_t id, int64_t schema_id,
@@ -120,9 +122,7 @@ class Snapshot : public Jsonizable {
              const std::optional& changelog_manifest_list_size,
              const std::optional& index_manifest, const std::string& commit_user,
              int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis,
-             const std::optional>& log_offsets,
-             const std::optional& total_record_count,
-             const std::optional& delta_record_count,
+             int64_t total_record_count, int64_t delta_record_count,
              const std::optional& changelog_record_count,
              const std::optional& watermark, const std::optional& statistics,
              const std::optional>& properties,
@@ -194,15 +194,11 @@ class Snapshot : public Jsonizable {
         return time_millis_;
     }
 
-    const std::optional>& LogOffsets() const {
-        return log_offsets_;
-    }
-
-    const std::optional& TotalRecordCount() const {
+    int64_t TotalRecordCount() const {
         return total_record_count_;
     }
 
-    const std::optional& DeltaRecordCount() const {
+    int64_t DeltaRecordCount() const {
         return delta_record_count_;
     }
 
@@ -275,15 +271,11 @@ class Snapshot : public Jsonizable {
 
     int64_t time_millis_;
 
-    std::optional> log_offsets_;
-
     // record count of all changes occurred in this snapshot
-    // null for paimon <= 0.3
-    std::optional total_record_count_;
+    int64_t total_record_count_ = 0;
 
     // record count of all new changes occurred in this snapshot
-    // null for paimon <= 0.3
-    std::optional delta_record_count_;
+    int64_t delta_record_count_ = 0;
 
     // record count of all changelog produced in this snapshot
     // null for paimon <= 0.3
diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp
index b5694e44..606e3200 100644
--- a/src/paimon/core/snapshot_test.cpp
+++ b/src/paimon/core/snapshot_test.cpp
@@ -34,12 +34,20 @@ class SnapshotTest : public testing::Test {
         std::string replaced_str = StringUtils::Replace(str, " ", "");
         replaced_str = StringUtils::Replace(replaced_str, "\t", "");
         replaced_str = StringUtils::Replace(replaced_str, "\n", "");
+        // logOffsets was removed from snapshot json; normalize legacy fixtures.
+        replaced_str = StringUtils::Replace(replaced_str, R"("logOffsets":{},)", "");
+        replaced_str = StringUtils::Replace(replaced_str, R"(,"logOffsets":{})", "");
+        replaced_str = StringUtils::Replace(replaced_str, R"("logOffsets":{})", "");
+        // changelogManifestList is @JsonInclude(NON_NULL) in Java and omitted on serialization;
+        // strip the stale null field from checked-in Java fixtures before comparing.
+        replaced_str = StringUtils::Replace(replaced_str, R"("changelogManifestList":null,)", "");
+        replaced_str = StringUtils::Replace(replaced_str, R"(,"changelogManifestList":null)", "");
+        replaced_str = StringUtils::Replace(replaced_str, R"("changelogManifestList":null)", "");
         return replaced_str;
     }
 };
 
 TEST_F(SnapshotTest, TestSimple) {
-    std::map log_offset = {{25, 30}};
     std::map properties = {{"key1", "value1"}, {"key2", "value2"}};
     Snapshot snapshot(
         /*version=*/5, /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"base_manifest_list", 10,
@@ -47,7 +55,7 @@ TEST_F(SnapshotTest, TestSimple) {
         /*changelog_manifest_list=*/"changelog_manifest_list", 30,
         /*index_manifest=*/"index_manifest",
         /*commit_user=*/"commit_user_01", /*commit_identifier=*/20,
-        /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset,
+        /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234,
         /*total_record_count=*/35,
         /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50,
         /*statistics=*/"statistic_test", properties, /*next_row_id=*/0);
@@ -65,9 +73,8 @@ TEST_F(SnapshotTest, TestSimple) {
     ASSERT_EQ(20, snapshot.CommitIdentifier());
     ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind());
     ASSERT_EQ(1234, snapshot.TimeMillis());
-    ASSERT_EQ(log_offset, snapshot.LogOffsets().value());
-    ASSERT_EQ(35, snapshot.TotalRecordCount().value());
-    ASSERT_EQ(40, snapshot.DeltaRecordCount().value());
+    ASSERT_EQ(35, snapshot.TotalRecordCount());
+    ASSERT_EQ(40, snapshot.DeltaRecordCount());
     ASSERT_EQ(45, snapshot.ChangelogRecordCount().value());
     ASSERT_EQ(50, snapshot.Watermark().value());
     ASSERT_EQ("statistic_test", snapshot.Statistics().value());
@@ -94,9 +101,8 @@ TEST_F(SnapshotTest, TestFromPath) {
     ASSERT_EQ(9223372036854775807ll, snapshot.CommitIdentifier());
     ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind());
     ASSERT_EQ(1721614343270ll, snapshot.TimeMillis());
-    ASSERT_EQ((std::map()), snapshot.LogOffsets().value());
-    ASSERT_EQ(5, snapshot.TotalRecordCount().value());
-    ASSERT_EQ(5, snapshot.DeltaRecordCount().value());
+    ASSERT_EQ(5, snapshot.TotalRecordCount());
+    ASSERT_EQ(5, snapshot.DeltaRecordCount());
     ASSERT_EQ(0, snapshot.ChangelogRecordCount().value());
     ASSERT_EQ(std::nullopt, snapshot.Watermark());
     ASSERT_EQ(std::nullopt, snapshot.Statistics());
@@ -113,12 +119,10 @@ TEST_F(SnapshotTest, TestJsonizable) {
         "baseManifestListSize" : 20,
         "deltaManifestList" : "manifest-list-d96fcc30-99e8-4f45-962b-a1157c56f378-1",
         "deltaManifestListSize" : 50,
-        "changelogManifestList" : null,
         "commitUser" : "0e4d92f7-53b0-40d6-a7c0-102bf3801e6a",
         "commitIdentifier" : 9223372036854775807,
         "commitKind" : "OVERWRITE",
         "timeMillis" : 1711692199281,
-        "logOffsets" : { },
         "totalRecordCount" : 3,
         "deltaRecordCount" : 3,
         "changelogRecordCount" : 0
@@ -135,7 +139,6 @@ TEST_F(SnapshotTest, TestJsonizable) {
         /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a",
         /*commit_identifier=*/9223372036854775807ll,
         /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll,
-        /*log_offsets=*/std::map(),
         /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0,
         /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt,
         /*next_row_id=*/std::nullopt);
@@ -193,15 +196,10 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) {
           "baseManifestListSize" : 100,
           "deltaManifestList" : "delta-manifest-list-2",
           "deltaManifestListSize" : 200,
-          "changelogManifestList" : null,
           "commitUser" : "commit-usr-3",
           "commitIdentifier" : 12,
           "commitKind" : "APPEND",
           "timeMillis" : 1749724197266,
-          "logOffsets" : {
-              "0" : 1,
-              "1" : 3
-          },
           "totalRecordCount" : 1024,
           "deltaRecordCount" : 4096,
           "watermark" : 1749724196266,
@@ -223,15 +221,10 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) {
           "baseManifestListSize" : 100,
           "deltaManifestList" : "delta-manifest-list-2",
           "deltaManifestListSize" : 200,
-          "changelogManifestList" : null,
           "commitUser" : "commit-usr-3",
           "commitIdentifier" : 12,
           "commitKind" : "APPEND",
           "timeMillis" : 1749724197266,
-          "logOffsets" : {
-              "0" : 1,
-              "1" : 3
-          },
           "totalRecordCount" : 1024,
           "deltaRecordCount" : 4096,
           "watermark" : 1749724196266,
@@ -261,7 +254,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyze) {
         /*commit_identifier=*/42,
         /*commit_kind=*/Snapshot::CommitKind::Analyze(),
         /*time_millis=*/1700000000000ll,
-        /*log_offsets=*/std::map(),
         /*total_record_count=*/0,
         /*delta_record_count=*/0,
         /*changelog_record_count=*/0,
@@ -283,12 +275,10 @@ TEST_F(SnapshotTest, TestCommitKindAnalyzeSerializeAndDeserialize) {
         "baseManifestListSize" : 100,
         "deltaManifestList" : "delta-manifest-analyze",
         "deltaManifestListSize" : 200,
-        "changelogManifestList" : null,
         "commitUser" : "analyze-user",
         "commitIdentifier" : 42,
         "commitKind" : "ANALYZE",
         "timeMillis" : 1700000000000,
-        "logOffsets" : { },
         "totalRecordCount" : 0,
         "deltaRecordCount" : 0,
         "changelogRecordCount" : 0,
@@ -354,7 +344,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) {
             "commitIdentifier" : 100,
             "commitKind" : "APPEND",
             "timeMillis" : 1700000000000,
-            "logOffsets" : { },
             "totalRecordCount" : 10,
             "deltaRecordCount" : 5,
             "changelogRecordCount" : 3
@@ -380,12 +369,10 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) {
             "schemaId" : 0,
             "baseManifestList" : "base-manifest-list",
             "deltaManifestList" : "delta-manifest-list",
-            "changelogManifestList" : null,
             "commitUser" : "user-02",
             "commitIdentifier" : 200,
             "commitKind" : "COMPACT",
             "timeMillis" : 1700000001000,
-            "logOffsets" : { },
             "totalRecordCount" : 20,
             "deltaRecordCount" : 10,
             "changelogRecordCount" : 0
diff --git a/src/paimon/core/stats/simple_stats_evolution.cpp b/src/paimon/core/stats/simple_stats_evolution.cpp
index a9356ed7..42177a0b 100644
--- a/src/paimon/core/stats/simple_stats_evolution.cpp
+++ b/src/paimon/core/stats/simple_stats_evolution.cpp
@@ -64,7 +64,6 @@ SimpleStatsEvolution::SimpleStatsEvolution(const std::vector& data_fi
         const auto& data_field = data_fields[i];
         id_to_data_fields_.emplace(data_field.Id(), std::make_pair(i, data_field));
     }
-    std::map name_to_table_fields;
     for (const auto& field : table_fields) {
         name_to_table_fields_.emplace(field.Name(), field);
     }
diff --git a/src/paimon/core/table/bucket_mode.cpp b/src/paimon/core/table/bucket_mode.cpp
new file mode 100644
index 00000000..d46739f8
--- /dev/null
+++ b/src/paimon/core/table/bucket_mode.cpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/bucket_mode.h"
+
+#include "paimon/core/schema/table_schema.h"
+
+namespace paimon {
+
+BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr& table_schema) {
+    if (bucket == BucketModeDefine::POSTPONE_BUCKET) {
+        return BucketMode::POSTPONE_MODE;
+    }
+    if (bucket == -1) {
+        return table_schema->PrimaryKeys().empty() ? BucketMode::BUCKET_UNAWARE
+                                                   : BucketMode::HASH_DYNAMIC;
+    }
+    if (bucket == BucketModeDefine::UNAWARE_BUCKET) {
+        return BucketMode::BUCKET_UNAWARE;
+    }
+    return BucketMode::HASH_FIXED;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/bucket_mode.h b/src/paimon/core/table/bucket_mode.h
index 2891dac7..f87ab346 100644
--- a/src/paimon/core/table/bucket_mode.h
+++ b/src/paimon/core/table/bucket_mode.h
@@ -19,9 +19,12 @@
 #pragma once
 
 #include 
+#include 
 
 namespace paimon {
 
+class TableSchema;
+
 /// Bucket mode of the table, it affects the writing process and also affects the data skipping in
 /// reading.
 enum class BucketMode {
@@ -64,4 +67,6 @@ class BucketModeDefine {
     static constexpr int32_t POSTPONE_BUCKET = -2;
 };
 
+BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr& table_schema);
+
 }  // namespace paimon
diff --git a/src/paimon/core/table/bucket_mode_test.cpp b/src/paimon/core/table/bucket_mode_test.cpp
new file mode 100644
index 00000000..2a77e5ea
--- /dev/null
+++ b/src/paimon/core/table/bucket_mode_test.cpp
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/bucket_mode.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "gtest/gtest.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+std::shared_ptr CreateTableSchema(const std::vector& primary_keys) {
+    auto schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false),
+                                 arrow::field("f1", arrow::int32(), /*nullable=*/false)});
+    std::vector partition_keys = {"f1"};
+    std::map options;
+    EXPECT_OK_AND_ASSIGN(
+        std::shared_ptr table_schema,
+        TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options));
+    return table_schema;
+}
+
+}  // namespace
+
+TEST(BucketModeTest, TestResolveBucketMode) {
+    std::shared_ptr append_schema = CreateTableSchema(/*primary_keys=*/{});
+    std::shared_ptr pk_schema = CreateTableSchema(/*primary_keys=*/{"f0"});
+
+    EXPECT_EQ(BucketMode::POSTPONE_MODE,
+              ResolveBucketMode(BucketModeDefine::POSTPONE_BUCKET, append_schema));
+    EXPECT_EQ(BucketMode::BUCKET_UNAWARE, ResolveBucketMode(-1, append_schema));
+    EXPECT_EQ(BucketMode::HASH_DYNAMIC, ResolveBucketMode(-1, pk_schema));
+    EXPECT_EQ(BucketMode::BUCKET_UNAWARE,
+              ResolveBucketMode(BucketModeDefine::UNAWARE_BUCKET, pk_schema));
+    EXPECT_EQ(BucketMode::HASH_FIXED, ResolveBucketMode(4, append_schema));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/table/source/append_count_reader.cpp b/src/paimon/core/table/source/append_count_reader.cpp
new file mode 100644
index 00000000..8f9af90a
--- /dev/null
+++ b/src/paimon/core/table/source/append_count_reader.cpp
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/append_count_reader.h"
+
+#include "paimon/core/deletionvectors/deletion_vector.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+Result AppendCountReader::CountRows() {
+    int64_t total = 0;
+    for (const auto& split : splits_) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t split_count, CountSingleSplit(split));
+        total += split_count;
+    }
+    return total;
+}
+
+Result AppendCountReader::CountSingleSplit(const std::shared_ptr& split) const {
+    auto data_split = std::dynamic_pointer_cast(split);
+    if (!data_split) {
+        return Status::Invalid("split cannot be cast to DataSplitImpl");
+    }
+
+    if (data_split->DataFiles().empty()) {
+        return 0;
+    }
+
+    return MetadataCount(data_split);
+}
+
+Result AppendCountReader::MetadataCount(
+    const std::shared_ptr& split) const {
+    if (split->RawConvertible()) {
+        if (!file_system_ || !pool_) {
+            return Status::Invalid(
+                "file_system or memory_pool is null for DV-based append count fallback");
+        }
+
+        DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
+            file_system_,
+            DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()),
+            pool_);
+
+        PAIMON_ASSIGN_OR_RAISE(std::optional merged_count,
+                               split->MergedRowCount(dv_factory));
+        if (merged_count.has_value()) {
+            return merged_count.value();
+        }
+    } else {
+        // Non-raw-convertible splits are typically produced by data evolution when multiple files
+        // overlap on row-id ranges. Count them through data-evolution metadata instead of using a
+        // deletion-vector factory for raw file counts.
+        PAIMON_ASSIGN_OR_RAISE(std::optional merged_count, split->MergedRowCount());
+        if (merged_count.has_value()) {
+            return merged_count.value();
+        }
+    }
+
+    return Status::Invalid("not support split in append count fallback");
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/append_count_reader.h b/src/paimon/core/table/source/append_count_reader.h
new file mode 100644
index 00000000..e6c6a096
--- /dev/null
+++ b/src/paimon/core/table/source/append_count_reader.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/reader/count_reader.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class DataSplitImpl;
+class Split;
+
+class AppendCountReader : public CountReader {
+ public:
+    explicit AppendCountReader(std::vector> splits,
+                               const std::shared_ptr& file_system,
+                               const std::shared_ptr& pool)
+        : splits_(std::move(splits)), file_system_(file_system), pool_(pool) {}
+
+    Result CountRows() override;
+
+ private:
+    Result CountSingleSplit(const std::shared_ptr& split) const;
+    Result MetadataCount(const std::shared_ptr& split) const;
+
+ private:
+    std::vector> splits_;
+    std::shared_ptr file_system_;
+    std::shared_ptr pool_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp
new file mode 100644
index 00000000..3b191a3d
--- /dev/null
+++ b/src/paimon/core/table/source/append_count_reader_test.cpp
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/append_count_reader.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/defs.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/scan_context.h"
+#include "paimon/status.h"
+#include "paimon/table/source/plan.h"
+#include "paimon/table/source/split.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+class DummySplit final : public Split {};
+
+}  // namespace
+
+class AppendCountReaderTest : public testing::Test {
+ protected:
+    Result>> CreateSplits(
+        const std::string& table_path, const std::optional& snapshot_id) {
+        ScanContextBuilder scan_context_builder(table_path);
+        if (snapshot_id.has_value()) {
+            scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID,
+                                           std::to_string(snapshot_id.value()));
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context)));
+        PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan());
+        if (snapshot_id.has_value() &&
+            (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) {
+            return Status::Invalid("snapshot id mismatch");
+        }
+        return plan->Splits();
+    }
+
+    std::shared_ptr file_system_ = std::make_shared();
+    std::shared_ptr pool_ = GetDefaultPool();
+};
+
+TEST_F(AppendCountReaderTest, TestCountRowsSnapshot1) {
+    std::string table_path = GetDataDir() + "/orc/append_09.db/append_09";
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/1));
+    AppendCountReader count_reader(splits, file_system_, pool_);
+
+    ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows());
+    ASSERT_EQ(count, 5);
+}
+
+TEST_F(AppendCountReaderTest, TestCountRowsSnapshot5) {
+    std::string table_path = GetDataDir() + "/orc/append_09.db/append_09";
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5));
+    AppendCountReader count_reader(splits, file_system_, pool_);
+
+    ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows());
+    ASSERT_EQ(count, 11);
+}
+
+TEST_F(AppendCountReaderTest, TestCountRowsDataEvolutionTable) {
+    std::string table_path =
+        GetDataDir() + "/orc/data_evolution_with_dense_stats.db/data_evolution_with_dense_stats";
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/2));
+    ASSERT_FALSE(splits.empty());
+
+    bool has_non_raw_convertible_split = false;
+    for (const auto& split : splits) {
+        auto data_split = std::dynamic_pointer_cast(split);
+        ASSERT_TRUE(data_split);
+        has_non_raw_convertible_split |= !data_split->RawConvertible();
+    }
+    ASSERT_TRUE(has_non_raw_convertible_split);
+
+    AppendCountReader count_reader(splits, file_system_, pool_);
+
+    ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows());
+    ASSERT_EQ(count, 2);
+}
+
+TEST_F(AppendCountReaderTest, TestCountRowsWithEmptySplits) {
+    std::vector> empty_splits;
+    AppendCountReader count_reader(empty_splits, file_system_, pool_);
+
+    ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows());
+    ASSERT_EQ(count, 0);
+}
+
+TEST_F(AppendCountReaderTest, TestCountRowsWithInvalidSplit) {
+    std::vector> splits = {std::make_shared()};
+    AppendCountReader count_reader(splits, file_system_, pool_);
+
+    ASSERT_NOK_WITH_MSG(count_reader.CountRows(), "split cannot be cast to DataSplitImpl");
+}
+
+TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateNotSupported) {
+    std::string table_path = GetDataDir() + "/orc/append_09.db/append_09";
+
+    auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3",
+                                                   FieldType::DOUBLE, Literal(13.0));
+
+    ReadContextBuilder read_context_builder(table_path);
+    read_context_builder.SetPredicate(predicate);
+    ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context)));
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5));
+    ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits),
+                        "predicate pushdown is not supported");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp
index b51dc839..eb6498c1 100644
--- a/src/paimon/core/table/source/append_only_table_read.cpp
+++ b/src/paimon/core/table/source/append_only_table_read.cpp
@@ -19,10 +19,13 @@
 
 #include "paimon/core/table/source/append_only_table_read.h"
 
+#include 
+
 #include "paimon/core/core_options.h"
 #include "paimon/core/operation/data_evolution_split_read.h"
 #include "paimon/core/operation/internal_read_context.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/table/source/append_count_reader.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -35,7 +38,7 @@ AppendOnlyTableRead::AppendOnlyTableRead(const std::shared_ptr& context,
                                          const std::shared_ptr& memory_pool,
                                          const std::shared_ptr& executor)
-    : TableRead(memory_pool) {
+    : TableRead(memory_pool), context_(context) {
     const auto& core_options = context->GetCoreOptions();
     if (core_options.DataEvolutionEnabled()) {
         // add data evolution first
@@ -58,4 +61,15 @@ Result> AppendOnlyTableRead::CreateReader(
     return Status::Invalid("create reader failed, not read match with split.");
 }
 
+Result> AppendOnlyTableRead::CreateCountReader(
+    const std::vector>& splits) {
+    if (context_->GetPredicate() != nullptr) {
+        return Status::NotImplemented(
+            "CreateCountReader with predicate pushdown is not supported yet");
+    }
+
+    return std::make_unique(splits, context_->GetCoreOptions().GetFileSystem(),
+                                               GetMemoryPool());
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/append_only_table_read.h b/src/paimon/core/table/source/append_only_table_read.h
index b911deca..e26b8613 100644
--- a/src/paimon/core/table/source/append_only_table_read.h
+++ b/src/paimon/core/table/source/append_only_table_read.h
@@ -47,8 +47,12 @@ class AppendOnlyTableRead : public TableRead {
     Result> CreateReader(
         const std::shared_ptr& data_split) override;
 
+    Result> CreateCountReader(
+        const std::vector>& splits) override;
+
  private:
     std::vector> split_reads_;
+    std::shared_ptr context_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp
index d43addfc..ed09b693 100644
--- a/src/paimon/core/table/source/data_evolution_batch_scan.cpp
+++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp
@@ -19,6 +19,9 @@
 
 #include "paimon/core/table/source/data_evolution_batch_scan.h"
 
+#include 
+#include 
+
 #include "paimon/core/global_index/global_index_scan_impl.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/table/source/data_split_impl.h"
@@ -75,7 +78,7 @@ Result> DataEvolutionBatchScan::CreatePlan() {
 
 Result> DataEvolutionBatchScan::WrapToIndexedSplits(
     const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index,
-    const std::map& id_to_score) const {
+    const std::map& id_to_score) {
     // TODO(lisizhuo.lsz): add executor here
     auto data_splits = data_plan->Splits();
     std::vector> indexed_splits;
@@ -89,11 +92,23 @@ Result> DataEvolutionBatchScan::WrapToIndexedSplits(
         if (files.empty()) {
             return Status::Invalid("Empty data files in WrapToIndexedSplits");
         }
-        PAIMON_ASSIGN_OR_RAISE(int64_t min, files[0]->NonNullFirstRowId());
-        PAIMON_ASSIGN_OR_RAISE(int64_t max, files[files.size() - 1]->NonNullFirstRowId());
-        max += files[files.size() - 1]->row_count - 1;
+        // The row-id ranges of the files in a split may be unordered, discontiguous, or
+        // overlapping, so intersect the index with each file's range separately, then sort and
+        // merge the intersected ranges.
+        std::vector intersected;
+        int64_t min = std::numeric_limits::max();
+        int64_t max = std::numeric_limits::min();
+        for (const auto& file : files) {
+            PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
+            int64_t last_row_id = first_row_id + file->row_count - 1;
+            min = std::min(min, first_row_id);
+            max = std::max(max, last_row_id);
+            std::vector file_ranges =
+                row_range_index.IntersectedRanges(first_row_id, last_row_id);
+            intersected.insert(intersected.end(), file_ranges.begin(), file_ranges.end());
+        }
 
-        std::vector expected = row_range_index.IntersectedRanges(min, max);
+        std::vector expected = Range::SortAndMergeOverlap(intersected, /*adjacent=*/true);
         if (expected.empty()) {
             return Status::Invalid(
                 fmt::format("There should be intersected ranges for split with min row id {} and "
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.h b/src/paimon/core/table/source/data_evolution_batch_scan.h
index 1dc2f296..cfa29785 100644
--- a/src/paimon/core/table/source/data_evolution_batch_scan.h
+++ b/src/paimon/core/table/source/data_evolution_batch_scan.h
@@ -19,6 +19,7 @@
 
 #pragma once
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -40,10 +41,14 @@ class DataEvolutionBatchScan : public AbstractTableScan {
 
     Result> CreatePlan() override;
 
- private:
-    Result> WrapToIndexedSplits(
+    /// Wraps each DataSplit in `data_plan` into an IndexedSplit whose row ranges are the
+    /// intersection of `row_range_index` and the row-id range of each data file. Visible for
+    /// testing.
+    static Result> WrapToIndexedSplits(
         const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index,
-        const std::map& id_to_score) const;
+        const std::map& id_to_score);
+
+ private:
     Result> EvalGlobalIndex() const;
 
  private:
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
new file mode 100644
index 00000000..14fae397
--- /dev/null
+++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/core/table/source/plan_impl.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/range.h"
+#include "paimon/utils/row_range_index.h"
+
+namespace paimon::test {
+namespace {
+std::shared_ptr NewAppendFile(const std::string& file_name, int64_t first_row_id,
+                                            int64_t row_count) {
+    return std::make_shared(
+        file_name, /*file_size=*/1024l, row_count, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
+        SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/0l,
+        /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0,
+        std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0,
+        /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id,
+        std::nullopt);
+}
+
+std::shared_ptr NewDataPlan(std::vector> files) {
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRow::EmptyRow(),
+        /*bucket=*/0, /*bucket_path=*/"data/test_table/bucket-0", std::move(files));
+    std::shared_ptr data_split =
+        builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value();
+    return std::make_shared(/*snapshot_id=*/1,
+                                      std::vector>({data_split}));
+}
+}  // namespace
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithUnorderedAndDiscontiguousDataFiles) {
+    // The files cover [4650, 4700], [4300, 4450] and [4200, 4407]: unordered, partially
+    // overlapping, with a gap [4451, 4649] that must not appear in the wrapped ranges.
+    std::shared_ptr data_plan =
+        NewDataPlan({NewAppendFile("file-1", 4650l, 51l), NewAppendFile("file-2", 4300l, 151l),
+                     NewAppendFile("file-3", 4200l, 208l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 5000)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr plan,
+                         DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index,
+                                                                     /*id_to_score=*/{}));
+
+    ASSERT_EQ(plan->Splits().size(), 1);
+    auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]);
+    ASSERT_NE(indexed_split, nullptr);
+    ASSERT_EQ(indexed_split->GetDataSplit(), data_plan->Splits()[0]);
+    ASSERT_EQ(indexed_split->RowRanges(),
+              std::vector({Range(4200, 4450), Range(4650, 4700)}));
+    ASSERT_TRUE(indexed_split->Scores().empty());
+}
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsExcludesRowIdsInFileRangeGaps) {
+    // The split covers [0, 9] and [20, 29] while the index hits {5, 15, 25}. Row id 15 lies in
+    // the gap between the two files and must be excluded, together with its score.
+    std::shared_ptr data_plan =
+        NewDataPlan({NewAppendFile("file-1", 0l, 10l), NewAppendFile("file-2", 20l, 10l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index,
+                         RowRangeIndex::Create({Range(5, 5), Range(15, 15), Range(25, 25)}));
+    std::map id_to_score = {{5l, 0.5f}, {15l, 0.7f}, {25l, 0.9f}};
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, DataEvolutionBatchScan::WrapToIndexedSplits(
+                                                         data_plan, row_range_index, id_to_score));
+
+    ASSERT_EQ(plan->Splits().size(), 1);
+    auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]);
+    ASSERT_NE(indexed_split, nullptr);
+    ASSERT_EQ(indexed_split->RowRanges(), std::vector({Range(5, 5), Range(25, 25)}));
+    ASSERT_EQ(indexed_split->Scores(), std::vector({0.5f, 0.9f}));
+}
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) {
+    std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 100l, 10l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 50)}));
+
+    ASSERT_NOK_WITH_MSG(DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index,
+                                                                    /*id_to_score=*/{}),
+                        "There should be intersected ranges for split with min row id 100 and "
+                        "max row id 109.");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/table/source/data_evolution_split_generator.cpp b/src/paimon/core/table/source/data_evolution_split_generator.cpp
index 66b705bc..0a76e75a 100644
--- a/src/paimon/core/table/source/data_evolution_split_generator.cpp
+++ b/src/paimon/core/table/source/data_evolution_split_generator.cpp
@@ -24,6 +24,7 @@
 #include 
 #include 
 
+#include "paimon/common/data/blob_utils.h"
 #include "paimon/common/utils/bin_packing.h"
 #include "paimon/common/utils/range_helper.h"
 #include "paimon/core/io/data_file_meta.h"
@@ -43,11 +44,15 @@ Result> DataEvolutionSplitGenerator::Spl
     PAIMON_ASSIGN_OR_RAISE(std::vector>> ranges,
                            range_helper.MergeOverlappingRanges(std::move(input)));
 
-    auto weight_func = [open_file_cost = open_file_cost_](
+    auto weight_func = [open_file_cost = open_file_cost_, count_blob_size = count_blob_size_](
                            const std::vector>& metas) -> int64_t {
         int64_t file_size_sum = 0;
         for (const auto& meta : metas) {
-            file_size_sum += meta->file_size;
+            if (BlobUtils::IsBlobFile(meta->file_name)) {
+                file_size_sum += count_blob_size ? meta->file_size : open_file_cost;
+            } else {
+                file_size_sum += meta->file_size;
+            }
         }
         return std::max(file_size_sum, open_file_cost);
     };
diff --git a/src/paimon/core/table/source/data_evolution_split_generator.h b/src/paimon/core/table/source/data_evolution_split_generator.h
index 7e9d75ca..73ccd211 100644
--- a/src/paimon/core/table/source/data_evolution_split_generator.h
+++ b/src/paimon/core/table/source/data_evolution_split_generator.h
@@ -33,8 +33,11 @@ struct DataFileMeta;
 /// Append data evolution table split generator, which implementation of `SplitGenerator`.
 class DataEvolutionSplitGenerator : public SplitGenerator {
  public:
-    DataEvolutionSplitGenerator(int64_t target_split_size, int64_t open_file_cost)
-        : target_split_size_(target_split_size), open_file_cost_(open_file_cost) {}
+    DataEvolutionSplitGenerator(int64_t target_split_size, int64_t open_file_cost,
+                                bool count_blob_size)
+        : target_split_size_(target_split_size),
+          open_file_cost_(open_file_cost),
+          count_blob_size_(count_blob_size) {}
 
     Result> SplitForBatch(
         std::vector>&& input) const override;
@@ -47,6 +50,7 @@ class DataEvolutionSplitGenerator : public SplitGenerator {
  private:
     int64_t target_split_size_;
     int64_t open_file_cost_;
+    bool count_blob_size_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/data_split_impl.cpp b/src/paimon/core/table/source/data_split_impl.cpp
index 020b5018..67abe964 100644
--- a/src/paimon/core/table/source/data_split_impl.cpp
+++ b/src/paimon/core/table/source/data_split_impl.cpp
@@ -24,6 +24,9 @@
 #include 
 #include 
 
+#include "paimon/common/utils/range_helper.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
+
 namespace paimon {
 
 bool DataSplit::SimpleDataFileMeta::operator==(const SimpleDataFileMeta& other) const {
@@ -112,18 +115,83 @@ bool DataSplitImpl::TEST_Equal(const DataSplitImpl& other) const {
            is_streaming_ == other.is_streaming_ && raw_convertible_ == other.raw_convertible_;
 }
 
-int64_t DataSplitImpl::PartialMergedRowCount() const {
+Result> DataSplitImpl::MergedRowCount() const {
+    return MergedRowCount(nullptr);
+}
+
+Result> DataSplitImpl::MergedRowCount(
+    DeletionVector::Factory dv_factory) const {
+    PAIMON_ASSIGN_OR_RAISE(std::optional raw_merged_row_count,
+                           RawMergedRowCount(dv_factory));
+    if (raw_merged_row_count.has_value()) {
+        return raw_merged_row_count;
+    }
+    if (DataEvolutionRowCountAvailable()) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t merged_row_count, DataEvolutionMergedRowCount());
+        return std::optional(merged_row_count);
+    }
+    return std::optional();
+}
+
+Result> DataSplitImpl::RawMergedRowCount(
+    DeletionVector::Factory dv_factory) const {
     if (!raw_convertible_) {
-        return 0;
+        return std::optional();
     }
+
     int64_t sum = 0;
     for (size_t i = 0; i < data_files_.size(); i++) {
         const auto& data_file = data_files_[i];
-        if (data_deletion_files_.empty() || data_deletion_files_[i] == std::nullopt) {
+        const std::optional deletion_file =
+            data_deletion_files_.empty() ? std::nullopt : data_deletion_files_[i];
+        if (deletion_file == std::nullopt) {
             sum += data_file->row_count;
-        } else if (data_deletion_files_[i].value().cardinality != std::nullopt) {
-            sum += data_file->row_count - data_deletion_files_[i].value().cardinality.value();
+        } else if (deletion_file.value().cardinality != std::nullopt) {
+            sum += data_file->row_count - deletion_file.value().cardinality.value();
+        } else {
+            if (!dv_factory) {
+                return std::optional();
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr deletion_vector,
+                                   dv_factory(data_file->file_name));
+            if (!deletion_vector) {
+                return Status::Invalid(
+                    "deletion vector not found for file with missing cardinality");
+            }
+            sum += data_file->row_count - deletion_vector->GetCardinality();
+        }
+    }
+
+    return std::optional(sum);
+}
+
+bool DataSplitImpl::DataEvolutionRowCountAvailable() const {
+    return std::all_of(data_files_.begin(), data_files_.end(),
+                       [](const std::shared_ptr& file) {
+                           return file->first_row_id != std::nullopt;
+                       });
+}
+
+Result DataSplitImpl::DataEvolutionMergedRowCount() const {
+    std::vector> files = data_files_;
+    RangeHelper> range_helper(
+        [](const std::shared_ptr& meta) -> Result {
+            return meta->first_row_id.value();
+        },
+        [](const std::shared_ptr& meta) -> Result {
+            return meta->first_row_id.value() + meta->row_count - 1;
+        });
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector>> ranges,
+                           range_helper.MergeOverlappingRanges(std::move(files)));
+
+    int64_t sum = 0;
+    for (const auto& group : ranges) {
+        int64_t max_count = 0;
+        for (const auto& file : group) {
+            max_count = std::max(max_count, file->row_count);
         }
+        sum += max_count;
     }
     return sum;
 }
diff --git a/src/paimon/core/table/source/data_split_impl.h b/src/paimon/core/table/source/data_split_impl.h
index ce1124ab..8dbe5377 100644
--- a/src/paimon/core/table/source/data_split_impl.h
+++ b/src/paimon/core/table/source/data_split_impl.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -28,6 +29,7 @@
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/preconditions.h"
 #include "paimon/common/utils/string_utils.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
 #include "paimon/core/io/data_file_meta_09_serializer.h"
 #include "paimon/core/io/data_file_meta_10_serializer.h"
 #include "paimon/core/io/data_file_meta_12_serializer.h"
@@ -37,6 +39,7 @@
 #include "paimon/table/source/data_split.h"
 
 namespace paimon {
+
 /// Input splits. Needed by most batch computation engines.
 class DataSplitImpl : public DataSplit {
  public:
@@ -51,7 +54,7 @@ class DataSplitImpl : public DataSplit {
         return partition_;
     }
 
-    int32_t Bucket() const {
+    int32_t Bucket() const override {
         return bucket_;
     }
 
@@ -96,14 +99,22 @@ class DataSplitImpl : public DataSplit {
     bool operator==(const DataSplitImpl& other) const;
     bool TEST_Equal(const DataSplitImpl& other) const;
 
-    /// Obtain merged row count as much as possible. There are two scenarios where accurate row
-    /// count
-    /// can be calculated:
+    /// Obtain merged row count when metadata is sufficient.
+    ///
+    /// This method follows Java DataSplit#mergedRowCount behavior:
     ///
-    /// 1. raw file and no deletion file.
+    /// 1. Prefer raw merged row count when split is raw-convertible and deletion cardinality is
+    ///    available.
+    /// 2. Fallback to data-evolution merged row count when all files have first_row_id.
     ///
-    /// 2. raw file + deletion file with cardinality.
-    int64_t PartialMergedRowCount() const;
+    /// Obtain merged row count without reading deletion vector files for missing cardinality.
+    Result> MergedRowCount() const;
+
+    /// Obtain merged row count with a deletion vector factory for missing cardinality.
+    ///
+    /// When a deletion file exists but its cardinality metadata is missing, the factory can be
+    /// used to read the deletion vector file and provide exact cardinality.
+    Result> MergedRowCount(DeletionVector::Factory dv_factory) const;
 
     // Builder
     /// Builder for `DataSplitImpl`.
@@ -177,6 +188,10 @@ class DataSplitImpl : public DataSplit {
           bucket_path_(bucket_path),
           data_files_(std::move(data_files)) {}
 
+    Result> RawMergedRowCount(DeletionVector::Factory dv_factory) const;
+    bool DataEvolutionRowCountAvailable() const;
+    Result DataEvolutionMergedRowCount() const;
+
  private:
     int64_t snapshot_id_ = 0;
     BinaryRow partition_ = BinaryRow::EmptyRow();
diff --git a/src/paimon/core/table/source/data_split_test.cpp b/src/paimon/core/table/source/data_split_test.cpp
index 528e19cd..6e69b038 100644
--- a/src/paimon/core/table/source/data_split_test.cpp
+++ b/src/paimon/core/table/source/data_split_test.cpp
@@ -28,6 +28,8 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/data_define.h"
+#include "paimon/core/deletionvectors/bitmap_deletion_vector.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/stats/simple_stats.h"
@@ -415,7 +417,9 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithEmptyExternalPath) {
         builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value());
 
     ASSERT_EQ(*result_data_split, *expected_data_split);
-    ASSERT_EQ(5, expected_data_split->PartialMergedRowCount());
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count,
+                         expected_data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(5), merged_row_count);
 
     ASSERT_OK(Split::Serialize(result_data_split, pool));
 }
@@ -477,7 +481,9 @@ TEST(DataSplitTest, TestDeserializeVersion4PkWithSnapshot4WithDvCardinality) {
             .value());
 
     ASSERT_EQ(*result_data_split, *expected_data_split);
-    ASSERT_EQ(5, expected_data_split->PartialMergedRowCount());
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count,
+                         expected_data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(5), merged_row_count);
 }
 
 TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1) {
@@ -816,7 +822,9 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfSingleFile) {
             .Build()
             .value());
     ASSERT_EQ(*result_data_split, *expected_data_split);
-    ASSERT_EQ(0, expected_data_split->PartialMergedRowCount());
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count,
+                         expected_data_split->MergedRowCount());
+    ASSERT_EQ(std::nullopt, merged_row_count);
 }
 
 TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) {
@@ -1051,16 +1059,8 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) {
 TEST(DataSplitTest, TestPartialMergedRowCount) {
     auto pool = GetDefaultPool();
     auto file_meta = std::make_shared(
-        "data-0.orc", /*file_size=*/100, /*row_count=*/2,
-        /*min_key=*/
-        BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool.get()), /*max_key=*/
-        BinaryRowGenerator::GenerateRow({std::string("David"), 1}, pool.get()),
-        /*key_stats=*/
-        BinaryRowGenerator::GenerateStats({std::string("Alice"), 1}, {std::string("David"), 1},
-                                          {0, 0}, pool.get()), /*value_stats=*/
-        BinaryRowGenerator::GenerateStats({std::string("Alice"), 10, 1, 11.0},
-                                          {std::string("David"), 10, 1, 11.1}, {0, 0, 0, 0},
-                                          pool.get()),
+        "data-0.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
         /*min_sequence_number=*/0, /*max_sequence_number=*/1, /*schema_id=*/0,
         /*level=*/0, /*extra_files=*/std::vector>(),
         /*creation_time=*/Timestamp(1725562946338ll, 0),
@@ -1069,16 +1069,8 @@ TEST(DataSplitTest, TestPartialMergedRowCount) {
         /*first_row_id=*/std::nullopt,
         /*write_cols=*/std::nullopt);
     auto file_meta2 = std::make_shared(
-        "data-1.orc", /*file_size=*/100, /*row_count=*/2,
-        /*min_key=*/
-        BinaryRowGenerator::GenerateRow({std::string("Bob"), 1}, pool.get()), /*max_key=*/
-        BinaryRowGenerator::GenerateRow({std::string("David"), 1}, pool.get()),
-        /*key_stats=*/
-        BinaryRowGenerator::GenerateStats({std::string("Bob"), 1}, {std::string("David"), 1},
-                                          {0, 0}, pool.get()), /*value_stats=*/
-        BinaryRowGenerator::GenerateStats({std::string("Bob"), 10, 1, 11.0},
-                                          {std::string("David"), 10, 1, 11.1}, {0, 0, 0, 0},
-                                          pool.get()),
+        "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
         /*min_sequence_number=*/2, /*max_sequence_number=*/3, /*schema_id=*/0,
         /*level=*/0, /*extra_files=*/std::vector>(),
         /*creation_time=*/Timestamp(1725562947338ll, 0),
@@ -1098,7 +1090,9 @@ TEST(DataSplitTest, TestPartialMergedRowCount) {
                                                      .RawConvertible(false)
                                                      .Build()
                                                      .value());
-    ASSERT_EQ(0, expected_data_split->PartialMergedRowCount());
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count,
+                         expected_data_split->MergedRowCount());
+    ASSERT_EQ(std::nullopt, merged_row_count);
 
     ASSERT_EQ(4, expected_data_split->RowCount());
     ASSERT_OK_AND_ASSIGN(auto latest_epoch, expected_data_split->LatestFileCreationEpochMillis());
@@ -1106,6 +1100,299 @@ TEST(DataSplitTest, TestPartialMergedRowCount) {
     ASSERT_EQ(file_meta2->CreationTimeEpochMillis().value(), latest_epoch.value());
 }
 
+TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/4, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/3, /*max_sequence_number=*/6, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2});
+
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value());
+
+    // Java parity: rawConvertible and deletion files absent should return row_count sum.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(7), merged_row_count);
+}
+
+TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/7, /*max_sequence_number=*/8, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta3 = std::make_shared(
+        "data-2.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/9, /*max_sequence_number=*/11, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562948338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0",
+        {file_meta1, file_meta2, file_meta3});
+
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1)
+            .WithDataDeletionFiles({std::nullopt,
+                                    DeletionFile("fake/index-0", /*offset=*/1, /*length=*/22,
+                                                 /*cardinality=*/2),
+                                    DeletionFile("fake/index-0", /*offset=*/31, /*length=*/22,
+                                                 /*cardinality=*/1)})
+            .IsStreaming(false)
+            .RawConvertible(true)
+            .Build()
+            .value());
+
+    // Java parity: null + cardinality + cardinality => (7 + (2 - 2) + (3 - 1)) = 9.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(9), merged_row_count);
+}
+
+TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/7, /*max_sequence_number=*/8, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2});
+
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1)
+            .WithDataDeletionFiles({DeletionFile("fake/index-0", /*offset=*/1, /*length=*/22,
+                                                 /*cardinality=*/2),
+                                    DeletionFile("fake/index-0", /*offset=*/31, /*length=*/22,
+                                                 /*cardinality=*/std::nullopt)})
+            .IsStreaming(false)
+            .RawConvertible(true)
+            .Build()
+            .value());
+
+    // If any deletion file misses cardinality, metadata count is unknown and must not be partial.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount());
+    ASSERT_EQ(std::nullopt, merged_row_count);
+}
+
+TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRawCount) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/3, /*max_sequence_number=*/4, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2});
+
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1)
+            .WithDataDeletionFiles({std::nullopt, std::nullopt})
+            .IsStreaming(false)
+            .RawConvertible(true)
+            .Build()
+            .value());
+
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(5), merged_row_count);
+}
+
+TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/100,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/3, /*max_sequence_number=*/7, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/100,
+        /*write_cols=*/std::nullopt);
+    auto file_meta3 = std::make_shared(
+        "data-2.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/8, /*max_sequence_number=*/9, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562948338ll, 0),
+        /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/200,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0",
+        {file_meta1, file_meta2, file_meta3});
+
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1)
+            .WithDataDeletionFiles({std::nullopt, std::nullopt, std::nullopt})
+            .IsStreaming(false)
+            .RawConvertible(false)
+            .Build()
+            .value());
+
+    // [100,102] and [100,104] overlap -> max row_count is 5; [200,201] is separate -> 2.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount());
+    ASSERT_EQ(std::optional(7), merged_row_count);
+}
+
+// Covers the refactored path where a deletion file exists but its cardinality metadata is
+// missing (nullopt). In that case MergedRowCount must call the provided dv_factory to read the
+// deletion vector and derive the exact cardinality, instead of returning nullopt.
+TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactory) {
+    auto pool = GetDefaultPool();
+    auto file_meta1 = std::make_shared(
+        "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562946338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+    auto file_meta2 = std::make_shared(
+        "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(),
+        DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+        /*min_sequence_number=*/7, /*max_sequence_number=*/11, /*schema_id=*/0,
+        /*level=*/0, /*extra_files=*/std::vector>(),
+        /*creation_time=*/Timestamp(1725562947338ll, 0),
+        /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
+        /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt,
+        /*first_row_id=*/std::nullopt,
+        /*write_cols=*/std::nullopt);
+
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()),
+        /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2});
+
+    // The second data file has a deletion file whose cardinality metadata is missing (nullopt),
+    // which forces resolution through the dv_factory.
+    auto data_split = std::dynamic_pointer_cast(
+        builder.WithSnapshot(1)
+            .WithDataDeletionFiles({std::nullopt, DeletionFile("fake/index-0", /*offset=*/0,
+                                                               /*length=*/22, /*cardinality=*/
+                                                               std::nullopt)})
+            .IsStreaming(false)
+            .RawConvertible(true)
+            .Build()
+            .value());
+
+    // Without a factory, the missing cardinality keeps the result unknown.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_without_factory,
+                         data_split->MergedRowCount());
+    ASSERT_EQ(std::nullopt, merged_without_factory);
+
+    // Build a deletion vector with 2 deleted rows; the factory returns it for the data file whose
+    // cardinality is missing.
+    RoaringBitmap32 roaring;
+    roaring.Add(1);
+    roaring.Add(3);
+    auto deletion_vector = std::make_shared(roaring);
+    ASSERT_EQ(2, deletion_vector->GetCardinality());
+
+    DeletionVector::Factory dv_factory =
+        [&deletion_vector](
+            const std::string& file_name) -> Result> {
+        if (file_name == "data-1.orc") {
+            return std::static_pointer_cast(deletion_vector);
+        }
+        return std::shared_ptr();
+    };
+
+    // file_meta1: 7 (no deletion file) + file_meta2: 5 - factory_cardinality(2) = 10.
+    ASSERT_OK_AND_ASSIGN(std::optional merged_with_factory,
+                         data_split->MergedRowCount(dv_factory));
+    ASSERT_EQ(std::optional(10), merged_with_factory);
+}
+
 TEST(DataSplitTest, TestRowCountAndLatestFileCreationEpochMillisEmpty) {
     DataSplitImpl::Builder builder(
         /*partition=*/BinaryRow::EmptyRow(),
diff --git a/src/paimon/core/table/source/data_table_batch_scan.cpp b/src/paimon/core/table/source/data_table_batch_scan.cpp
index 9cb816f5..d45a140c 100644
--- a/src/paimon/core/table/source/data_table_batch_scan.cpp
+++ b/src/paimon/core/table/source/data_table_batch_scan.cpp
@@ -36,10 +36,15 @@ class DataSplit;
 
 DataTableBatchScan::DataTableBatchScan(bool pk_table, const CoreOptions& core_options,
                                        const std::shared_ptr& snapshot_reader,
-                                       std::optional push_down_limit)
+                                       bool read_optimized, std::optional push_down_limit)
     : AbstractTableScan(core_options, snapshot_reader), push_down_limit_(push_down_limit) {
-    if (pk_table && (core_options.DeletionVectorsEnabled() ||
-                     core_options.GetMergeEngine() == MergeEngine::FIRST_ROW)) {
+    if (pk_table && read_optimized) {
+        int32_t top_level = core_options.GetNumLevels() - 1;
+        snapshot_reader_->WithLevelFilter(
+            [top_level](int32_t level) -> bool { return level == top_level; });
+        snapshot_reader_->EnableValueFilter();
+    } else if (pk_table && (core_options.DeletionVectorsEnabled() ||
+                            core_options.GetMergeEngine() == MergeEngine::FIRST_ROW)) {
         auto level_filter = [](int32_t level) -> bool { return level > 0; };
         snapshot_reader_->WithLevelFilter(level_filter);
         snapshot_reader_->EnableValueFilter();
@@ -83,9 +88,14 @@ Result> DataTableBatchScan::ApplyPushDownLimit(
             return Status::Invalid("DataSplit cannot cast to DataSplitImpl");
         }
         if (data_split->RawConvertible()) {
-            int64_t partial_merged_row_count = data_split->PartialMergedRowCount();
+            PAIMON_ASSIGN_OR_RAISE(std::optional partial_merged_row_count,
+                                   data_split->MergedRowCount());
+            if (!partial_merged_row_count.has_value()) {
+                // Cannot safely estimate split rows from metadata; skip push-down limit.
+                return current_scan_result->GetPlan();
+            }
             limited_data_splits.emplace_back(data_split);
-            scanned_row_count += partial_merged_row_count;
+            scanned_row_count += partial_merged_row_count.value();
             if (scanned_row_count >= push_down_limit_.value()) {
                 PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, current_scan_result->SnapshotId());
                 return std::make_shared(snapshot_id, limited_data_splits);
diff --git a/src/paimon/core/table/source/data_table_batch_scan.h b/src/paimon/core/table/source/data_table_batch_scan.h
index 7f6dbb15..0729bd69 100644
--- a/src/paimon/core/table/source/data_table_batch_scan.h
+++ b/src/paimon/core/table/source/data_table_batch_scan.h
@@ -35,7 +35,7 @@ class SnapshotReader;
 class DataTableBatchScan : public AbstractTableScan {
  public:
     DataTableBatchScan(bool pk_table, const CoreOptions& core_options,
-                       const std::shared_ptr& snapshot_reader,
+                       const std::shared_ptr& snapshot_reader, bool read_optimized,
                        std::optional push_down_limit);
 
     Result> CreatePlan() override;
diff --git a/src/paimon/core/table/source/fallback_data_split.h b/src/paimon/core/table/source/fallback_data_split.h
index c8d23987..897d5fad 100644
--- a/src/paimon/core/table/source/fallback_data_split.h
+++ b/src/paimon/core/table/source/fallback_data_split.h
@@ -30,6 +30,10 @@ class FallbackDataSplit : public DataSplit {
     FallbackDataSplit(const std::shared_ptr& split, bool is_fallback)
         : is_fallback_(is_fallback), split_(split) {}
 
+    int32_t Bucket() const override {
+        return split_->Bucket();
+    }
+
     std::vector GetFileList() const override {
         return split_->GetFileList();
     }
diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp
index 0d709094..23b90226 100644
--- a/src/paimon/core/table/source/key_value_table_read.cpp
+++ b/src/paimon/core/table/source/key_value_table_read.cpp
@@ -23,6 +23,8 @@
 
 #include "paimon/core/operation/merge_file_split_read.h"
 #include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/core/table/source/pk_count_reader.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -33,8 +35,15 @@ class InternalReadContext;
 class MemoryPool;
 
 KeyValueTableRead::KeyValueTableRead(std::vector>&& split_reads,
-                                     const std::shared_ptr& memory_pool)
-    : TableRead(memory_pool), split_reads_(std::move(split_reads)) {}
+                                     const std::shared_ptr& path_factory,
+                                     const std::shared_ptr& context,
+                                     const std::shared_ptr& memory_pool,
+                                     const std::shared_ptr& executor)
+    : TableRead(memory_pool),
+      split_reads_(std::move(split_reads)),
+      path_factory_(path_factory),
+      context_(context),
+      executor_(executor) {}
 
 Result> KeyValueTableRead::Create(
     const std::shared_ptr& path_factory,
@@ -49,7 +58,8 @@ Result> KeyValueTableRead::Create(
         MergeFileSplitRead::Create(path_factory, context, memory_pool, executor));
     split_reads.emplace_back(std::move(merge_file_split_read));
 
-    return std::unique_ptr(new KeyValueTableRead(std::move(split_reads), memory_pool));
+    return std::unique_ptr(new KeyValueTableRead(std::move(split_reads), path_factory,
+                                                            context, memory_pool, executor));
 }
 
 void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) {
@@ -77,4 +87,22 @@ Result> KeyValueTableRead::CreateReader(
     return Status::Invalid("create reader failed, not read match with data split.");
 }
 
+Result> KeyValueTableRead::CreateCountReader(
+    const std::vector>& splits) {
+    if (context_->GetPredicate() != nullptr) {
+        return Status::NotImplemented(
+            "CreateCountReader with predicate pushdown is not supported yet");
+    }
+
+    if (force_keep_delete_) {
+        return Status::NotImplemented("CreateCountReader with force_keep_delete is not supported");
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr pk_count_reader,
+        PKCountReader::Create(splits, path_factory_, context_, GetMemoryPool(), executor_));
+
+    return pk_count_reader;
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h
index 7751e599..d6a1c83d 100644
--- a/src/paimon/core/table/source/key_value_table_read.h
+++ b/src/paimon/core/table/source/key_value_table_read.h
@@ -45,13 +45,22 @@ class KeyValueTableRead : public TableRead {
 
     Result> CreateReader(const std::shared_ptr& split) override;
 
+    Result> CreateCountReader(
+        const std::vector>& splits) override;
+
     void ForceKeepDelete(bool force_keep_delete);
 
  private:
     KeyValueTableRead(std::vector>&& split_reads,
-                      const std::shared_ptr& memory_pool);
+                      const std::shared_ptr& path_factory,
+                      const std::shared_ptr& context,
+                      const std::shared_ptr& memory_pool,
+                      const std::shared_ptr& executor);
 
     std::vector> split_reads_;
+    std::shared_ptr path_factory_;
+    std::shared_ptr context_;
+    std::shared_ptr executor_;
     bool force_keep_delete_ = false;
 };
 
diff --git a/src/paimon/core/table/source/pk_count_reader.cpp b/src/paimon/core/table/source/pk_count_reader.cpp
new file mode 100644
index 00000000..2e492ad7
--- /dev/null
+++ b/src/paimon/core/table/source/pk_count_reader.cpp
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/pk_count_reader.h"
+
+#include 
+#include 
+#include 
+
+#include "arrow/c/abi.h"
+#include "arrow/type.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/core/deletionvectors/deletion_vector.h"
+#include "paimon/core/mergetree/compact/interval_partition.h"
+#include "paimon/core/mergetree/row_count_accumulator.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/operation/merge_file_split_read.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+PKCountReader::~PKCountReader() = default;
+
+Result> PKCountReader::Create(
+    std::vector> splits,
+    const std::shared_ptr& path_factory,
+    const std::shared_ptr& context,
+    const std::shared_ptr& memory_pool, const std::shared_ptr& executor) {
+    const auto& table_schema = context->GetTableSchema();
+    PAIMON_ASSIGN_OR_RAISE(std::vector pk_fields,
+                           table_schema->TrimmedPrimaryKeyFields());
+    std::shared_ptr count_read_schema =
+        DataField::ConvertDataFieldsToArrowSchema(pk_fields);
+
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr count_context,
+                           InternalReadContext::CreateWithSchema(context, count_read_schema));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr merge_read,
+        MergeFileSplitRead::Create(path_factory, count_context, memory_pool, executor));
+
+    return std::unique_ptr(
+        new PKCountReader(std::move(splits), count_context, std::move(merge_read), memory_pool));
+}
+
+Result PKCountReader::CountRows() {
+    int64_t total = 0;
+    for (const auto& split : splits_) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t split_count, CountSingleSplit(split));
+        total += split_count;
+    }
+    return total;
+}
+
+PKCountReader::PKCountReader(std::vector> splits,
+                             const std::shared_ptr& context,
+                             std::unique_ptr&& merge_read,
+                             const std::shared_ptr& memory_pool)
+    : splits_(std::move(splits)),
+      context_(context),
+      merge_read_(std::move(merge_read)),
+      pool_(memory_pool) {}
+
+Result PKCountReader::CountSingleSplit(const std::shared_ptr& split) {
+    auto data_split = std::dynamic_pointer_cast(split);
+    if (!data_split) {
+        return Status::Invalid("split cannot be cast to DataSplitImpl");
+    }
+
+    if (data_split->DataFiles().empty()) {
+        return 0;
+    }
+
+    if (data_split->RawConvertible()) {
+        return MetadataCount(data_split);
+    }
+
+    return MergeCount(data_split);
+}
+
+Result PKCountReader::MetadataCount(const std::shared_ptr& split) {
+    DeletionVector::Factory dv_factory = DeletionVector::CreateFactory(
+        context_->GetCoreOptions().GetFileSystem(),
+        DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()), pool_);
+
+    PAIMON_ASSIGN_OR_RAISE(std::optional count, split->MergedRowCount(dv_factory));
+    if (count.has_value()) {
+        return count.value();
+    }
+
+    return Status::Invalid("not support split in pk count metadata fallback");
+}
+
+Result PKCountReader::MergeCount(const std::shared_ptr& split) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
+                           merge_read_->GetPathFactory()->CreateDataFilePathFactory(
+                               split->Partition(), split->Bucket()));
+
+    auto dv_factory = DeletionVector::CreateFactory(
+        context_->GetCoreOptions().GetFileSystem(),
+        DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()), pool_);
+
+    std::vector> sections =
+        IntervalPartition(split->DataFiles(), merge_read_->GetKeyComparator()).Partition();
+
+    int64_t total_count = 0;
+
+    for (const auto& section : sections) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_reader,
+                               merge_read_->CreateSortMergeReaderForSection(
+                                   section, split->Partition(), dv_factory,
+                                   /*predicate=*/nullptr, data_file_path_factory,
+                                   /*drop_delete=*/true));
+
+        RowCountAccumulator accumulator(std::move(merged_reader));
+        PAIMON_ASSIGN_OR_RAISE(int64_t section_count, accumulator.CountAll());
+        total_count += section_count;
+        accumulator.Close();
+    }
+
+    return total_count;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/pk_count_reader.h b/src/paimon/core/table/source/pk_count_reader.h
new file mode 100644
index 00000000..701c6121
--- /dev/null
+++ b/src/paimon/core/table/source/pk_count_reader.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "paimon/reader/count_reader.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+class DataSplitImpl;
+class Executor;
+class FileStorePathFactory;
+class InternalReadContext;
+class MemoryPool;
+class MergeFileSplitRead;
+class Split;
+
+class PKCountReader : public CountReader {
+ public:
+    ~PKCountReader() override;
+
+    static Result> Create(
+        std::vector> splits,
+        const std::shared_ptr& path_factory,
+        const std::shared_ptr& context,
+        const std::shared_ptr& memory_pool, const std::shared_ptr& executor);
+
+    Result CountRows() override;
+
+ private:
+    PKCountReader(std::vector> splits,
+                  const std::shared_ptr& context,
+                  std::unique_ptr&& merge_read,
+                  const std::shared_ptr& memory_pool);
+
+    Result CountSingleSplit(const std::shared_ptr& split);
+    Result MetadataCount(const std::shared_ptr& split);
+    Result MergeCount(const std::shared_ptr& split);
+
+ private:
+    std::vector> splits_;
+    std::shared_ptr context_;
+    std::unique_ptr merge_read_;
+    std::shared_ptr pool_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp
new file mode 100644
index 00000000..72507fee
--- /dev/null
+++ b/src/paimon/core/table/source/pk_count_reader_test.cpp
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/pk_count_reader.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/defs.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/read_context.h"
+#include "paimon/scan_context.h"
+#include "paimon/table/source/plan.h"
+#include "paimon/table/source/split.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+class DummySplit final : public Split {};
+
+}  // namespace
+
+class PKCountReaderTest : public testing::Test {
+ protected:
+    Result>> CreateSplits(
+        const std::string& table_path, const std::optional& snapshot_id) {
+        ScanContextBuilder scan_context_builder(table_path);
+        if (snapshot_id.has_value()) {
+            scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID,
+                                           std::to_string(snapshot_id.value()));
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish());
+        PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context)));
+        PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan());
+        if (snapshot_id.has_value()) {
+            if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) {
+                return Status::Invalid("snapshot id mismatch");
+            }
+        }
+        return plan->Splits();
+    }
+
+    Result> CreateInternalContext(
+        const std::string& table_path) {
+        ReadContextBuilder read_context_builder(table_path);
+        PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish());
+
+        SchemaManager schema_manager(std::make_shared(), table_path);
+        PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0));
+        PAIMON_ASSIGN_OR_RAISE(auto internal_context,
+                               InternalReadContext::Create(std::move(read_context), table_schema,
+                                                           table_schema->Options()));
+        return std::shared_ptr(std::move(internal_context));
+    }
+
+    Result> CreatePathFactory(
+        const std::shared_ptr& internal_context) {
+        const auto& core_options = internal_context->GetCoreOptions();
+        auto arrow_schema =
+            DataField::ConvertDataFieldsToArrowSchema(internal_context->GetTableSchema()->Fields());
+
+        PAIMON_ASSIGN_OR_RAISE(std::vector external_paths,
+                               core_options.CreateExternalPaths());
+        PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path,
+                               core_options.CreateGlobalIndexExternalPath());
+
+        PAIMON_ASSIGN_OR_RAISE(
+            auto path_factory,
+            FileStorePathFactory::Create(
+                internal_context->GetPath(), arrow_schema,
+                internal_context->GetTableSchema()->PartitionKeys(),
+                core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(),
+                core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
+                external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
+                pool_));
+
+        return std::shared_ptr(std::move(path_factory));
+    }
+
+ protected:
+    std::shared_ptr pool_ = GetDefaultPool();
+};
+
+TEST_F(PKCountReaderTest, TestCountRowsWithMORSnapshot5) {
+    std::string table_path =
+        GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/";
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5));
+    ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path));
+    ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context));
+
+    ASSERT_OK_AND_ASSIGN(auto pk_count_reader,
+                         PKCountReader::Create(splits, path_factory, internal_context, pool_,
+                                               internal_context->GetExecutor()));
+    ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows());
+
+    ASSERT_EQ(count, 11);
+}
+
+TEST_F(PKCountReaderTest, TestCountRowsWithDVSnapshot6) {
+    std::string table_path =
+        GetDataDir() + "/orc/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/";
+
+    ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/6));
+    ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path));
+    ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context));
+
+    ASSERT_OK_AND_ASSIGN(auto pk_count_reader,
+                         PKCountReader::Create(splits, path_factory, internal_context, pool_,
+                                               internal_context->GetExecutor()));
+    ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows());
+
+    ASSERT_EQ(count, 8);
+}
+
+TEST_F(PKCountReaderTest, TestCountRowsWithEmptySplits) {
+    std::string table_path =
+        GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/";
+
+    ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path));
+    ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context));
+
+    std::vector> empty_splits;
+    ASSERT_OK_AND_ASSIGN(auto pk_count_reader,
+                         PKCountReader::Create(empty_splits, path_factory, internal_context, pool_,
+                                               internal_context->GetExecutor()));
+    ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows());
+
+    ASSERT_EQ(count, 0);
+}
+
+TEST_F(PKCountReaderTest, TestCountRowsWithInvalidSplit) {
+    std::string table_path =
+        GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/";
+
+    ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path));
+    ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context));
+
+    std::vector> splits = {std::make_shared()};
+    ASSERT_OK_AND_ASSIGN(auto pk_count_reader,
+                         PKCountReader::Create(splits, path_factory, internal_context, pool_,
+                                               internal_context->GetExecutor()));
+
+    ASSERT_NOK_WITH_MSG(pk_count_reader->CountRows(), "split cannot be cast to DataSplitImpl");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/table/source/read_optimized_scan_options.h b/src/paimon/core/table/source/read_optimized_scan_options.h
new file mode 100644
index 00000000..150a5ea3
--- /dev/null
+++ b/src/paimon/core/table/source/read_optimized_scan_options.h
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+namespace paimon {
+
+/// Internal scan option used by `T$ro` to request top-level-only planning.
+inline constexpr char kReadOptimizedScanOption[] = "__paimon.internal.read-optimized";
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader.cpp
index c6b0477e..0e98b5cf 100644
--- a/src/paimon/core/table/source/snapshot/snapshot_reader.cpp
+++ b/src/paimon/core/table/source/snapshot/snapshot_reader.cpp
@@ -123,8 +123,8 @@ Result>> SnapshotReader::GetDeletionFile
         if (dv_metas != std::nullopt) {
             for (const auto& dv_meta_iter : dv_metas.value()) {
                 const auto& dv_meta = dv_meta_iter.second;
-                data_file_to_index_file_meta.insert(
-                    std::make_pair(dv_meta.GetDataFileName(), index_file_meta));
+                data_file_to_index_file_meta.insert_or_assign(dv_meta.GetDataFileName(),
+                                                              index_file_meta);
             }
         }
     }
diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
new file mode 100644
index 00000000..cebc644c
--- /dev/null
+++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "gtest/gtest.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+class SnapshotReaderTest : public testing::Test {
+ protected:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        dir_ = UniqueTestDirectory::Create();
+        ASSERT_TRUE(dir_ != nullptr);
+    }
+
+    std::shared_ptr CreateDataFileMeta(const std::string& file_name) const {
+        return std::make_shared(
+            file_name, /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(),
+            DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(),
+            /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0,
+            DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0),
+            std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt,
+            std::nullopt);
+    }
+
+    std::shared_ptr CreateIndexFileMeta(const std::string& index_file_name,
+                                                       const std::string& data_file_name,
+                                                       int64_t offset, int64_t length,
+                                                       std::optional cardinality) const {
+        LinkedHashMap dv_ranges;
+        dv_ranges.insert(data_file_name,
+                         DeletionVectorMeta(data_file_name, offset, length, cardinality));
+        return std::make_shared(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX,
+                                               index_file_name, /*file_size=*/100, /*row_count=*/10,
+                                               dv_ranges, /*external_path=*/std::nullopt);
+    }
+
+    Result> CreateIndexFileHandler() const {
+        auto schema = arrow::schema({arrow::field("f0", arrow::int32())});
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr path_factory,
+            FileStorePathFactory::Create(
+                dir_->Str(), schema, /*partition_keys=*/{}, /*default_part_value=*/"", "orc",
+                /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true,
+                /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt,
+                /*index_file_in_data_file_dir=*/false, pool_));
+        auto path_factories = std::make_shared(path_factory);
+        return std::make_unique(std::make_shared(),
+                                                  std::unique_ptr(),
+                                                  path_factories, /*dv_bitmap64=*/false, pool_);
+    }
+
+    std::shared_ptr pool_;
+    std::unique_ptr dir_;
+};
+
+TEST_F(SnapshotReaderTest, GetDeletionFilesOverwritesDuplicateDataFileName) {
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler,
+                         CreateIndexFileHandler());
+    SnapshotReader snapshot_reader(/*scan=*/nullptr, /*path_factory=*/nullptr,
+                                   /*split_generator=*/nullptr, std::move(index_file_handler));
+
+    const std::string data_file_name = "data-0.orc";
+    std::vector> data_files = {CreateDataFileMeta(data_file_name)};
+    std::vector> index_file_metas = {
+        CreateIndexFileMeta("index-first", data_file_name, /*offset=*/1, /*length=*/11,
+                            /*cardinality=*/3),
+        CreateIndexFileMeta("index-second", data_file_name, /*offset=*/2, /*length=*/22,
+                            /*cardinality=*/4)};
+
+    ASSERT_OK_AND_ASSIGN(std::vector> deletion_files,
+                         snapshot_reader.GetDeletionFiles(BinaryRow::EmptyRow(), /*bucket=*/0,
+                                                          data_files, index_file_metas));
+
+    ASSERT_EQ(deletion_files.size(), 1);
+    ASSERT_TRUE(deletion_files[0].has_value());
+    EXPECT_EQ(deletion_files[0]->path, dir_->Str() + "/index/index-second");
+    EXPECT_EQ(deletion_files[0]->offset, 2);
+    EXPECT_EQ(deletion_files[0]->length, 22);
+    EXPECT_EQ(deletion_files[0]->cardinality, std::optional(4));
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/core/table/source/split_generator_test.cpp b/src/paimon/core/table/source/split_generator_test.cpp
index caf9e701..ac80eb3f 100644
--- a/src/paimon/core/table/source/split_generator_test.cpp
+++ b/src/paimon/core/table/source/split_generator_test.cpp
@@ -39,6 +39,7 @@
 #include "paimon/core/stats/simple_stats.h"
 #include "paimon/core/table/bucket_mode.h"
 #include "paimon/core/table/source/append_only_split_generator.h"
+#include "paimon/core/table/source/data_evolution_split_generator.h"
 #include "paimon/core/table/source/merge_tree_split_generator.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/memory/memory_pool.h"
@@ -119,6 +120,21 @@ class SplitGeneratorTest : public testing::Test {
             /*write_cols=*/std::nullopt);
     }
 
+    std::shared_ptr CreateDataFileMetaWithRowId(const std::string& file_name,
+                                                              int64_t file_size, int64_t row_count,
+                                                              int64_t first_row_id) {
+        return std::make_shared(
+            file_name, file_size, row_count, /*min_key=*/BinaryRow::EmptyRow(),
+            /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(),
+            /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0,
+            /*max_sequence_number=*/0, /*schema_id=*/0,
+            /*level=*/0, /*extra_files=*/std::vector>(),
+            /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0,
+            /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt,
+            /*external_path=*/std::nullopt, first_row_id,
+            /*write_cols=*/std::nullopt);
+    }
+
     static void CheckResult(const std::vector& result_groups,
                             const std::vector>& expected_file_names,
                             const std::vector& expected_raw_convertible) {
@@ -209,6 +225,41 @@ TEST_F(SplitGeneratorTest, TestAppend) {
     }
 }
 
+TEST_F(SplitGeneratorTest, TestDataEvolutionBlobSplitByFileSize) {
+    // two row id ranges, each with one data file and one much larger blob file
+    auto create_files = [&]() {
+        return std::vector>{
+            CreateDataFileMetaWithRowId("f1", /*file_size=*/100, /*row_count=*/100,
+                                        /*first_row_id=*/0),
+            CreateDataFileMetaWithRowId("blob1.blob", /*file_size=*/1000, /*row_count=*/100,
+                                        /*first_row_id=*/0),
+            CreateDataFileMetaWithRowId("f2", /*file_size=*/100, /*row_count=*/100,
+                                        /*first_row_id=*/100),
+            CreateDataFileMetaWithRowId("blob2.blob", /*file_size=*/1000, /*row_count=*/100,
+                                        /*first_row_id=*/100)};
+    };
+    {
+        // blob file size counts in splitting: each range weighs 1100, so the two ranges cannot
+        // be packed into one split of target size 1200
+        DataEvolutionSplitGenerator split_generator(/*target_split_size=*/1200,
+                                                    /*open_file_cost=*/10,
+                                                    /*count_blob_size=*/true);
+        ASSERT_OK_AND_ASSIGN(std::vector split_groups,
+                             split_generator.SplitForBatch(create_files()));
+        CheckResult(split_groups, {{"f1", "blob1.blob"}, {"f2", "blob2.blob"}}, {false, false});
+    }
+    {
+        // blob file only weighs the open file cost: each range weighs 110, both ranges fit in
+        // one split of target size 1200
+        DataEvolutionSplitGenerator split_generator(/*target_split_size=*/1200,
+                                                    /*open_file_cost=*/10,
+                                                    /*count_blob_size=*/false);
+        ASSERT_OK_AND_ASSIGN(std::vector split_groups,
+                             split_generator.SplitForBatch(create_files()));
+        CheckResult(split_groups, {{"f1", "blob1.blob", "f2", "blob2.blob"}}, {false});
+    }
+}
+
 TEST_F(SplitGeneratorTest, TestMergeTree) {
     std::vector> files = {
         CreateDataFileMeta("1", 0, 10),  CreateDataFileMeta("2", 0, 12),
diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp
index 5af23cf6..e7e3f02f 100644
--- a/src/paimon/core/table/source/table_read.cpp
+++ b/src/paimon/core/table/source/table_read.cpp
@@ -181,4 +181,10 @@ Result> TableRead::CreateReader(
     return std::make_unique(std::move(batch_readers), pool_);
 }
 
+Result> TableRead::CreateCountReader(
+    const std::vector>& splits) {
+    (void)splits;
+    return Status::NotImplemented("CreateCountReader is not implemented for this table type");
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp
index 691e4d79..bc2c7f6f 100644
--- a/src/paimon/core/table/source/table_read_test.cpp
+++ b/src/paimon/core/table/source/table_read_test.cpp
@@ -44,7 +44,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) {
     {
         // read with non-exist field
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f0", "f1", "non-exist"});
+        context_builder.SetReadFieldNames({"f0", "f1", "non-exist"});
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)),
                             "Get field non-exist failed: not exist in table schema");
@@ -75,7 +75,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) {
         auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f3",
                                                  FieldType::DOUBLE, Literal(15.0));
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f3", "f0", "f1"});
+        context_builder.SetReadFieldNames({"f3", "f0", "f1"});
         context_builder.SetPredicate(predicate);
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         ASSERT_NOK_WITH_MSG(
@@ -95,7 +95,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) {
     {
         // schema with duplicate field f3
         ReadContextBuilder context_builder(path);
-        context_builder.SetReadSchema({"f3", "f1", "f3"});
+        context_builder.SetReadFieldNames({"f3", "f1", "f3"});
         ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
         ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)),
                             "validate schema failed: read schema has duplicate field f3");
@@ -105,7 +105,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) {
 TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) {
     std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"field_no_exist"});
+    context_builder.SetReadFieldNames({"field_no_exist"});
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
     ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)),
                         "Get field field_no_exist failed: not exist in table schema");
@@ -115,7 +115,7 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) {
     std::string path = paimon::test::GetDataDir() +
                        "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f0", "f1", "f2", "f3"});
+    context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"});
     context_builder.AddOption("read.batch-size", "2");
     context_builder.AddOption("orc.read.enable-lazy-decoding", "true");
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -127,7 +127,7 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) {
 TEST(TableReadTest, TestCreateAppendOnlyTableRead) {
     std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f0", "f1", "f2", "f3"});
+    context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"});
     context_builder.AddOption("read.batch-size", "2");
     context_builder.AddOption("orc.read.enable-lazy-decoding", "true");
     ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
@@ -139,7 +139,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) {
 TEST(TableReadTest, TestMergeOptions) {
     std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
     ReadContextBuilder context_builder(path);
-    context_builder.SetReadSchema({"f0", "f1", "f2", "f3"});
+    context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"});
     context_builder.AddOption("read.batch-size", "2");
     context_builder.AddOption("orc.read.enable-lazy-decoding", "true");
     context_builder.AddOption("bucket", "10");
diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp
index 5afe9938..17add044 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -20,6 +20,7 @@
 #include "paimon/table/source/table_scan.h"
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -29,6 +30,7 @@
 #include "paimon/common/predicate/predicate_validator.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/options_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/index/index_file_handler.h"
 #include "paimon/core/manifest/index_manifest_file.h"
@@ -49,6 +51,7 @@
 #include "paimon/core/table/source/data_table_batch_scan.h"
 #include "paimon/core/table/source/data_table_stream_scan.h"
 #include "paimon/core/table/source/merge_tree_split_generator.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
 #include "paimon/core/table/source/snapshot/snapshot_reader.h"
 #include "paimon/core/table/source/split_generator.h"
 #include "paimon/core/table/system/system_table.h"
@@ -89,7 +92,7 @@ class TableScanImpl {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr manifest_list,
             ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(),
-                                 path_factory, memory_pool));
+                                 path_factory, core_options.GetCache(), memory_pool));
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr partition_schema,
             FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys()));
@@ -98,19 +101,35 @@ class TableScanImpl {
             ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(),
                                  path_factory, core_options.GetManifestTargetFileSize(),
                                  memory_pool, core_options, partition_schema));
+        std::unique_ptr scan;
         if (table_schema->PrimaryKeys().empty()) {
             if (core_options.DataEvolutionEnabled()) {
-                return DataEvolutionFileStoreScan::Create(
-                    snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
-                    arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool);
+                PAIMON_ASSIGN_OR_RAISE(
+                    scan, DataEvolutionFileStoreScan::Create(
+                              snapshot_manager, schema_manager, manifest_list, manifest_file,
+                              table_schema, arrow_schema, context->GetScanFilters(), core_options,
+                              executor, memory_pool));
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(
+                    scan, AppendOnlyFileStoreScan::Create(
+                              snapshot_manager, schema_manager, manifest_list, manifest_file,
+                              table_schema, arrow_schema, context->GetScanFilters(), core_options,
+                              executor, memory_pool));
             }
-            return AppendOnlyFileStoreScan::Create(
-                snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
-                arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool);
+        } else {
+            PAIMON_ASSIGN_OR_RAISE(
+                scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list,
+                                                    manifest_file, table_schema, arrow_schema,
+                                                    context->GetScanFilters(), core_options,
+                                                    executor, memory_pool));
         }
-        return KeyValueFileStoreScan::Create(
-            snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema,
-            arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool);
+        return WithTablePath(std::move(scan), context);
+    }
+
+    static std::unique_ptr WithTablePath(std::unique_ptr&& scan,
+                                                        const ScanContext* context) {
+        scan->WithTablePath(context->GetPath());
+        return std::move(scan);
     }
 
     static Result> CreateSplitGenerator(
@@ -120,8 +139,9 @@ class TableScanImpl {
         auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost();
         if (table_schema->PrimaryKeys().empty()) {
             if (core_options.DataEvolutionEnabled()) {
-                return std::make_unique(source_split_target_size,
-                                                                     source_split_open_file_cost);
+                return std::make_unique(
+                    source_split_target_size, source_split_open_file_cost,
+                    core_options.BlobSplitByFileSize());
             }
             BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE
                                                                      : BucketMode::HASH_FIXED);
@@ -177,7 +197,7 @@ Result> TableScan::Create(std::unique_ptrGetOptions(),
-                                                shared_context->GetSpecificFileSystem()));
+                                                shared_context->GetSpecificFileSystem(), {}));
     PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path,
                            SystemTableLoader::TryParsePath(shared_context->GetPath()));
     if (system_table_path) {
@@ -195,27 +215,33 @@ namespace {
 Result> NewDataTableScan(const std::shared_ptr& context) {
     PAIMON_ASSIGN_OR_RAISE(
         CoreOptions tmp_options,
-        CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem()));
+        CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {}));
     std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch());
-    SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch);
-    PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema,
-                           schema_manager.Latest());
-    if (latest_table_schema == std::nullopt) {
-        return Status::Invalid("not found latest schema");
-    }
-    const auto& table_schema = latest_table_schema.value();
-    if (table_schema->Id() != TableSchema::FIRST_SCHEMA_ID &&
-        !table_schema->PrimaryKeys().empty()) {
-        return Status::NotImplemented(
-            "do not support schema evolution in pk table while scan process");
+    std::shared_ptr table_schema;
+    const auto& specific_table_schema = context->GetSpecificTableSchema();
+    if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) {
+        PAIMON_ASSIGN_OR_RAISE(table_schema,
+                               TableSchema::CreateFromJson(specific_table_schema.value()));
+    } else {
+        SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch);
+        PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema,
+                               schema_manager.Latest());
+        if (latest_table_schema == std::nullopt) {
+            return Status::Invalid("not found latest schema");
+        }
+        table_schema = latest_table_schema.value();
     }
+    PAIMON_ASSIGN_OR_RAISE(bool read_optimized,
+                           OptionsUtils::GetValueFromMap(context->GetOptions(),
+                                                               kReadOptimizedScanOption, false));
     // merge options
     auto options = table_schema->Options();
     for (const auto& [key, value] : context->GetOptions()) {
         options[key] = value;
     }
     PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
-                           CoreOptions::FromMap(options, context->GetSpecificFileSystem()));
+                           CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {}));
+    core_options.WithCache(context->GetCache());
     // validate options
     if (core_options.GetBucket() == -1) {
         if (!table_schema->PrimaryKeys().empty()) {
@@ -263,12 +289,16 @@ Result> NewDataTableScan(const std::shared_ptrGetMemoryPool()));
     auto snapshot_reader = std::make_shared(
         file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler));
+    const bool pk_table = !table_schema->PrimaryKeys().empty();
+    if (read_optimized && pk_table && context->IsStreamingMode()) {
+        return Status::NotImplemented(
+            "read-optimized system table does not support streaming scan for primary key table");
+    }
     if (context->IsStreamingMode()) {
         return std::make_unique(core_options, snapshot_reader);
     }
-    auto batch_scan =
-        std::make_unique(/*pk_table=*/!table_schema->PrimaryKeys().empty(),
-                                             core_options, snapshot_reader, context->GetLimit());
+    auto batch_scan = std::make_unique(
+        /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit());
     if (!core_options.DataEvolutionEnabled()) {
         return batch_scan;
     }
diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp
index 8e43ad46..358dfd46 100644
--- a/src/paimon/core/table/source/table_scan_test.cpp
+++ b/src/paimon/core/table/source/table_scan_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/table/source/table_scan.h"
 
+#include 
 #include 
 #include 
 #include 
@@ -50,14 +51,29 @@ TEST(TableScanTest, TestNonExistTable) {
     ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema");
 }
 
-TEST(TableScanTest, TestNoSchemaEvolution) {
-    // do not bear schema evolution in scan
+TEST(TableScanTest, TestPkSchemaEvolutionScan) {
     std::string path =
         paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/";
     ScanContextBuilder builder(path);
     builder.AddOption(Options::FILE_FORMAT, "orc");
     ASSERT_OK_AND_ASSIGN(auto context, builder.Finish());
-    ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "do not support schema evolution");
+    ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context)));
+    ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan());
+    ASSERT_TRUE(plan->SnapshotId());
+    ASSERT_FALSE(plan->Splits().empty());
+}
+
+TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) {
+    std::string path = paimon::test::GetDataDir() +
+                       "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro";
+    ScanContextBuilder builder(path);
+    builder.AddOption(Options::FILE_FORMAT, "orc");
+    builder.WithStreamingMode(true);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish());
+
+    ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)),
+                        "read-optimized system table does not support streaming scan for primary "
+                        "key table");
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp
index bc330d99..e77821fe 100644
--- a/src/paimon/core/table/system/audit_log_system_table.cpp
+++ b/src/paimon/core/table/system/audit_log_system_table.cpp
@@ -34,6 +34,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/util/checked_cast.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
@@ -41,6 +42,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/key_value_table_read.h"
 #include "paimon/defs.h"
 #include "paimon/read_context.h"
@@ -55,7 +57,9 @@ namespace {
 class AuditLogBatchConverter : public ChangelogBatchConverter {
  public:
     Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* /*pool*/) const override {
+        const std::shared_ptr& array,
+        const std::vector& /*row_group_lengths*/,
+        arrow::MemoryPool* /*pool*/) const override {
         return array;
     }
 };
@@ -65,64 +69,76 @@ class ChangelogBatchReader : public BatchReader {
     ChangelogBatchReader(std::unique_ptr reader,
                          std::shared_ptr output_schema, bool include_sequence_number,
                          std::shared_ptr converter,
-                         const std::shared_ptr& pool)
+                         bool pack_update_before_after, const std::shared_ptr& pool)
         : reader_(std::move(reader)),
           output_schema_(std::move(output_schema)),
           include_sequence_number_(include_sequence_number),
           converter_(std::move(converter)),
+          pack_update_before_after_(pack_update_before_after),
           arrow_pool_holder_(GetArrowPool(pool)),
           arrow_pool_(arrow_pool_holder_.get()) {}
 
     Result NextBatch() override {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return batch;
-        }
-        auto& [c_array, c_schema] = batch;
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array,
-                                          arrow::ImportArray(c_array.get(), c_schema.get()));
-        std::shared_ptr struct_array =
-            std::dynamic_pointer_cast(arrow_array);
-        if (!struct_array) {
-            return Status::Invalid("audit_log system table expects struct batches");
-        }
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+            std::shared_ptr struct_array;
+            std::vector row_group_lengths;
+            if (BatchReader::IsEofBatch(batch)) {
+                if (!pending_update_before_) {
+                    return batch;
+                }
+                struct_array = std::move(pending_update_before_);
+                row_group_lengths.push_back(1);
+            } else {
+                auto& [c_array, c_schema] = batch;
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr arrow_array,
+                    arrow::ImportArray(c_array.get(), c_schema.get()));
+                struct_array = std::dynamic_pointer_cast(arrow_array);
+                if (!struct_array) {
+                    return Status::Invalid("audit_log system table expects struct batches");
+                }
+                PAIMON_ASSIGN_OR_RAISE(struct_array, PrependPendingUpdateBefore(struct_array));
+                PAIMON_ASSIGN_OR_RAISE(row_group_lengths, BuildRowGroupLengths(struct_array));
+            }
+            if (row_group_lengths.empty()) {
+                continue;
+            }
 
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rowkind_array,
-                               BuildRowKindArray(struct_array));
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_kind_array,
+                                   BuildRowKindArray(struct_array, row_group_lengths));
 
-        arrow::ArrayVector output_arrays = {rowkind_array};
-        if (include_sequence_number_) {
-            std::shared_ptr sequence_array =
-                struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name());
-            if (!sequence_array) {
-                return Status::Invalid("cannot find _SEQUENCE_NUMBER in audit_log batch");
+            arrow::ArrayVector output_arrays = {row_kind_array};
+            if (include_sequence_number_) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sequence_array,
+                                       BuildSequenceNumberArray(struct_array, row_group_lengths));
+                output_arrays.push_back(sequence_array);
             }
-            PAIMON_ASSIGN_OR_RAISE(sequence_array, CopyToStablePool(sequence_array));
-            output_arrays.push_back(sequence_array);
-        }
 
-        for (const auto& field : output_schema_->fields()) {
-            if (field->name() == SpecialFields::RowKind().Name() ||
-                field->name() == SpecialFields::SequenceNumber().Name()) {
-                continue;
+            for (const auto& field : output_schema_->fields()) {
+                if (field->name() == SpecialFields::RowKind().Name() ||
+                    field->name() == SpecialFields::SequenceNumber().Name()) {
+                    continue;
+                }
+                std::shared_ptr array = struct_array->GetFieldByName(field->name());
+                if (!array) {
+                    return Status::Invalid("cannot find ", field->name(), " in changelog batch");
+                }
+                PAIMON_ASSIGN_OR_RAISE(
+                    array, converter_->ConvertDataColumn(array, row_group_lengths, arrow_pool_));
+                PAIMON_ASSIGN_OR_RAISE(array, CopyToStablePool(array));
+                output_arrays.push_back(array);
             }
-            std::shared_ptr array = struct_array->GetFieldByName(field->name());
-            if (!array) {
-                return Status::Invalid("cannot find ", field->name(), " in changelog batch");
-            }
-            PAIMON_ASSIGN_OR_RAISE(array, converter_->ConvertDataColumn(array, arrow_pool_));
-            PAIMON_ASSIGN_OR_RAISE(array, CopyToStablePool(array));
-            output_arrays.push_back(array);
-        }
 
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr output_array,
-            arrow::StructArray::Make(output_arrays, output_schema_->field_names()));
-        auto output_c_array = std::make_unique();
-        auto output_c_schema = std::make_unique();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportArray(*output_array, output_c_array.get(), output_c_schema.get()));
-        return std::make_pair(std::move(output_c_array), std::move(output_c_schema));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr output_array,
+                arrow::StructArray::Make(output_arrays, output_schema_->field_names()));
+            auto output_c_array = std::make_unique();
+            auto output_c_schema = std::make_unique();
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                arrow::ExportArray(*output_array, output_c_array.get(), output_c_schema.get()));
+            return std::make_pair(std::move(output_c_array), std::move(output_c_schema));
+        }
     }
 
     std::shared_ptr GetReaderMetrics() const override {
@@ -130,21 +146,80 @@ class ChangelogBatchReader : public BatchReader {
     }
 
     void Close() override {
+        pending_update_before_.reset();
         reader_->Close();
     }
 
  private:
+    Result> PrependPendingUpdateBefore(
+        const std::shared_ptr& struct_array) {
+        if (!pending_update_before_) {
+            return struct_array;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr combined,
+            arrow::Concatenate({pending_update_before_, struct_array}, arrow_pool_));
+        pending_update_before_.reset();
+        std::shared_ptr result =
+            std::dynamic_pointer_cast(combined);
+        if (!result) {
+            return Status::Invalid("failed to concatenate binlog struct batches");
+        }
+        return result;
+    }
+
+    Result> BuildRowGroupLengths(
+        const std::shared_ptr& struct_array) {
+        std::shared_ptr value_kind_array =
+            std::dynamic_pointer_cast(
+                struct_array->GetFieldByName(SpecialFields::ValueKind().Name()));
+        if (!value_kind_array) {
+            return Status::Invalid("cannot find _VALUE_KIND in audit_log batch");
+        }
+
+        std::vector row_group_lengths;
+        row_group_lengths.reserve(struct_array->length());
+        for (int64_t i = 0; i < value_kind_array->length();) {
+            bool is_update_before =
+                !value_kind_array->IsNull(i) &&
+                value_kind_array->Value(i) == RowKind::UpdateBefore()->ToByteValue();
+            if (!pack_update_before_after_ || !is_update_before) {
+                row_group_lengths.push_back(1);
+                ++i;
+                continue;
+            }
+            if (i + 1 == value_kind_array->length()) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr pending,
+                                       CopyToStablePool(struct_array->Slice(i, /*length=*/1)));
+                pending_update_before_ = std::dynamic_pointer_cast(pending);
+                if (!pending_update_before_) {
+                    return Status::Invalid("failed to cache UPDATE_BEFORE in binlog reader");
+                }
+                break;
+            }
+            if (value_kind_array->IsNull(i + 1) ||
+                value_kind_array->Value(i + 1) != RowKind::UpdateAfter()->ToByteValue()) {
+                return Status::Invalid(
+                    "UPDATE_BEFORE is not followed by UPDATE_AFTER in binlog reader");
+            }
+            row_group_lengths.push_back(2);
+            i += 2;
+        }
+        return row_group_lengths;
+    }
+
     Result> CopyToStablePool(
         const std::shared_ptr& array) const {
-        // The imported data batch may release its C Arrow buffers after this wrapper returns.
-        // Keep returned system-table arrays independent of that input batch lifetime.
+        /// The imported data batch may release its C Arrow buffers after this wrapper returns.
+        /// Keep returned system-table arrays independent of that input batch lifetime.
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result,
                                           arrow::Concatenate({array}, arrow_pool_));
         return result;
     }
 
     Result> BuildRowKindArray(
-        const std::shared_ptr& struct_array) const {
+        const std::shared_ptr& struct_array,
+        const std::vector& row_group_lengths) const {
         std::shared_ptr value_kind_array =
             std::dynamic_pointer_cast(
                 struct_array->GetFieldByName(SpecialFields::ValueKind().Name()));
@@ -152,15 +227,46 @@ class ChangelogBatchReader : public BatchReader {
             return Status::Invalid("cannot find _VALUE_KIND in audit_log batch");
         }
         arrow::StringBuilder builder(arrow_pool_);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(value_kind_array->length()));
-        for (int64_t i = 0; i < value_kind_array->length(); ++i) {
-            if (value_kind_array->IsNull(i)) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(row_group_lengths.size()));
+        int64_t offset = 0;
+        for (int32_t row_group_length : row_group_lengths) {
+            int64_t row_kind_index = offset + row_group_length - 1;
+            if (value_kind_array->IsNull(row_kind_index)) {
+                return Status::Invalid(
+                    fmt::format("exists null value in value kind array in pos {}", row_kind_index));
                 PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull());
-                continue;
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(
+                    const RowKind* row_kind,
+                    RowKind::FromByteValue(value_kind_array->Value(row_kind_index)));
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(row_kind->ShortString()));
+            }
+            offset += row_group_length;
+        }
+        std::shared_ptr result;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result));
+        return result;
+    }
+
+    Result> BuildSequenceNumberArray(
+        const std::shared_ptr& struct_array,
+        const std::vector& row_group_lengths) const {
+        std::shared_ptr sequence_array =
+            std::dynamic_pointer_cast(
+                struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+        if (!sequence_array) {
+            return Status::Invalid("cannot find _SEQUENCE_NUMBER in audit_log batch");
+        }
+        arrow::Int64Builder builder(arrow_pool_);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(row_group_lengths.size()));
+        int64_t offset = 0;
+        for (int32_t row_group_length : row_group_lengths) {
+            if (sequence_array->IsNull(offset)) {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull());
+            } else {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(sequence_array->Value(offset)));
             }
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                                   RowKind::FromByteValue(value_kind_array->Value(i)));
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(row_kind->ShortString()));
+            offset += row_group_length;
         }
         std::shared_ptr result;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result));
@@ -171,8 +277,10 @@ class ChangelogBatchReader : public BatchReader {
     std::shared_ptr output_schema_;
     bool include_sequence_number_;
     std::shared_ptr converter_;
+    bool pack_update_before_after_;
     std::unique_ptr arrow_pool_holder_;
     arrow::MemoryPool* arrow_pool_;
+    std::shared_ptr pending_update_before_;
 };
 
 class ChangelogTableRead : public TableRead {
@@ -189,17 +297,42 @@ class ChangelogTableRead : public TableRead {
 
     Result> CreateReader(
         const std::vector>& splits) override {
+        // Records across different splits should not be packed, because for streaming reads on a
+        // primary-key table, all data belonging to the same partition and bucket is placed in a
+        // single split. Therefore, an UPDATE_BEFORE/UPDATE_AFTER pair will not be truncated at a
+        // split boundary.
+        if (converter_->PackUpdateBeforeAfter()) {
+            std::vector> readers;
+            readers.reserve(splits.size());
+            for (const auto& split : splits) {
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, CreateReader(split));
+                readers.push_back(std::move(reader));
+            }
+            return std::make_unique(std::move(readers), GetMemoryPool());
+        }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
                                data_read_->CreateReader(splits));
-        return std::make_unique(std::move(reader), output_schema_,
-                                                      include_sequence_number_, converter_,
-                                                      GetMemoryPool());
+        return CreateChangelogBatchReader(std::move(reader), output_schema_,
+                                          include_sequence_number_, converter_,
+                                          /*pack_update_before_after=*/false, GetMemoryPool());
     }
 
     Result> CreateReader(
         const std::shared_ptr& split) override {
-        std::vector> splits = {split};
-        return CreateReader(splits);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
+                               data_read_->CreateReader(split));
+        bool pack_update_before_after = false;
+        if (converter_->PackUpdateBeforeAfter()) {
+            std::shared_ptr data_split =
+                std::dynamic_pointer_cast(split);
+            if (!data_split) {
+                return Status::Invalid("binlog system table expects data split");
+            }
+            pack_update_before_after = data_split->IsStreaming();
+        }
+        return CreateChangelogBatchReader(std::move(reader), output_schema_,
+                                          include_sequence_number_, converter_,
+                                          pack_update_before_after, GetMemoryPool());
     }
 
  private:
@@ -211,6 +344,15 @@ class ChangelogTableRead : public TableRead {
 
 }  // namespace
 
+std::unique_ptr CreateChangelogBatchReader(
+    std::unique_ptr reader, std::shared_ptr output_schema,
+    bool include_sequence_number, std::shared_ptr converter,
+    bool pack_update_before_after, const std::shared_ptr& pool) {
+    return std::make_unique(std::move(reader), std::move(output_schema),
+                                                  include_sequence_number, std::move(converter),
+                                                  pack_update_before_after, pool);
+}
+
 AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path,
                                          std::shared_ptr table_schema,
                                          std::map options)
@@ -224,10 +366,10 @@ std::string AuditLogSystemTable::Name() const {
 }
 
 Result> AuditLogSystemTable::ArrowSchema() const {
-    std::shared_ptr rowkind_field =
+    std::shared_ptr row_kind_field =
         DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind());
-    rowkind_field = rowkind_field->WithNullable(false);
-    arrow::FieldVector fields = {rowkind_field};
+    row_kind_field = row_kind_field->WithNullable(false);
+    arrow::FieldVector fields = {row_kind_field};
     PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
     if (core_options.TableReadSequenceNumberEnabled()) {
         fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
@@ -249,7 +391,8 @@ Result> AuditLogSystemTable::NewScan(
         .WithStreamingMode(context->IsStreamingMode())
         .WithMemoryPool(context->GetMemoryPool())
         .WithExecutor(context->GetExecutor())
-        .WithFileSystem(context->GetSpecificFileSystem());
+        .WithFileSystem(context->GetSpecificFileSystem())
+        .WithCache(context->GetCache());
     if (scan_filter) {
         if (scan_filter->GetBucketFilter()) {
             builder.SetBucketFilter(scan_filter->GetBucketFilter().value());
@@ -286,7 +429,7 @@ Result> AuditLogSystemTable::NewChangelogRead(
     PAIMON_ASSIGN_OR_RAISE(StringMap read_options, ReadOptions());
     PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options));
     builder.SetOptions(read_options)
-        .SetReadSchema(base_read_schema->field_names())
+        .SetReadFieldNames(base_read_schema->field_names())
         .WithBranch(core_options.GetBranch())
         .WithMemoryPool(context->GetMemoryPool())
         .WithExecutor(context->GetExecutor())
@@ -298,7 +441,8 @@ Result> AuditLogSystemTable::NewChangelogRead(
         .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch())
         .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber())
         .SetPrefetchCacheMode(context->GetPrefetchCacheMode())
-        .WithCacheConfig(context->GetCacheConfig());
+        .WithCacheConfig(context->GetCacheConfig())
+        .WithCache(context->GetCache());
 
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish());
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read,
diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h
index df1829ae..e21e287c 100644
--- a/src/paimon/core/table/system/audit_log_system_table.h
+++ b/src/paimon/core/table/system/audit_log_system_table.h
@@ -30,14 +30,26 @@ namespace paimon {
 class FileSystem;
 class TableSchema;
 
+/// Converts data columns when wrapping the base table changelog reader.
 class ChangelogBatchConverter {
  public:
     virtual ~ChangelogBatchConverter() = default;
 
     virtual Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* pool) const = 0;
+        const std::shared_ptr& array, const std::vector& row_group_lengths,
+        arrow::MemoryPool* pool) const = 0;
+
+    virtual bool PackUpdateBeforeAfter() const {
+        return false;
+    }
 };
 
+std::unique_ptr CreateChangelogBatchReader(
+    std::unique_ptr reader, std::shared_ptr output_schema,
+    bool include_sequence_number, std::shared_ptr converter,
+    bool pack_update_before_after, const std::shared_ptr& pool);
+
+/// System table for `T$audit_log`, exposing row-level changelog records with rowkind.
 class AuditLogSystemTable : public SystemTable {
  public:
     static constexpr const char* kName = "audit_log";
diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp
index 57f05e2b..88541194 100644
--- a/src/paimon/core/table/system/binlog_system_table.cpp
+++ b/src/paimon/core/table/system/binlog_system_table.cpp
@@ -41,11 +41,15 @@ namespace {
 class BinlogBatchConverter : public ChangelogBatchConverter {
  public:
     Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* pool) const override {
+        const std::shared_ptr& array, const std::vector& row_group_lengths,
+        arrow::MemoryPool* pool) const override {
         arrow::Int32Builder offsets_builder(pool);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(array->length() + 1));
-        for (int64_t i = 0; i <= array->length(); ++i) {
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(static_cast(i)));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(row_group_lengths.size() + 1));
+        int32_t offset = 0;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
+        for (int32_t row_group_length : row_group_lengths) {
+            offset += row_group_length;
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
         }
         std::shared_ptr offsets_array;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets_array));
@@ -54,10 +58,18 @@ class BinlogBatchConverter : public ChangelogBatchConverter {
             arrow::ListArray::FromArrays(*offsets_array, *array, pool));
         return list_array;
     }
+
+    bool PackUpdateBeforeAfter() const override {
+        return true;
+    }
 };
 
 }  // namespace
 
+std::shared_ptr CreateBinlogBatchConverter() {
+    return std::make_shared();
+}
+
 BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path,
                                      std::shared_ptr table_schema,
                                      std::map options)
@@ -87,7 +99,7 @@ Result> BinlogSystemTable::ArrowSchema() const {
 
 Result> BinlogSystemTable::NewRead(
     const std::shared_ptr& context) const {
-    return NewChangelogRead(context, std::make_shared());
+    return NewChangelogRead(context, CreateBinlogBatchConverter());
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h
index 6ba95862..3b4f45bd 100644
--- a/src/paimon/core/table/system/binlog_system_table.h
+++ b/src/paimon/core/table/system/binlog_system_table.h
@@ -29,6 +29,9 @@ namespace paimon {
 class FileSystem;
 class TableSchema;
 
+std::shared_ptr CreateBinlogBatchConverter();
+
+/// System table for `T$binlog`, exposing changelog records with list-wrapped data columns.
 class BinlogSystemTable : public AuditLogSystemTable {
  public:
     static constexpr const char* kName = "binlog";
diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp
new file mode 100644
index 00000000..bdfab0f0
--- /dev/null
+++ b/src/paimon/core/table/system/global_system_tables.cpp
@@ -0,0 +1,543 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/system/global_system_tables.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/data/generic_row.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_entry.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_file.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/defs.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
+
+namespace paimon {
+namespace {
+
+// =============================================================================
+// Registry
+// =============================================================================
+
+using GlobalSystemTableFactory =
+    std::function>(const GlobalSystemTableContext&)>;
+
+struct GlobalSystemTableRegistryEntry {
+    std::string name;
+    bool requires_catalog;
+    GlobalSystemTableFactory factory;
+};
+
+const std::vector& GlobalSystemTableRegistry() {
+    static const std::vector registry = {
+        {CatalogOptionsSystemTable::kName, false,
+         [](const GlobalSystemTableContext& ctx) -> Result> {
+             return std::make_shared(ctx);
+         }},
+        {AllTableOptionsSystemTable::kName, true,
+         [](const GlobalSystemTableContext& ctx) -> Result> {
+             return std::make_shared(ctx);
+         }},
+        {TablesSystemTable::kName, true,
+         [](const GlobalSystemTableContext& ctx) -> Result> {
+             return std::make_shared(ctx);
+         }},
+        {PartitionsSystemTable::kName, true,
+         [](const GlobalSystemTableContext& ctx) -> Result> {
+             return std::make_shared(ctx);
+         }},
+    };
+    return registry;
+}
+
+// =============================================================================
+// Helpers for sys.tables and sys.partitions
+// =============================================================================
+
+VariantType StringValue(const std::string& value) {
+    return BinaryString::FromString(value, GetDefaultPool().get());
+}
+
+VariantType OptionalStringValue(const std::map& options,
+                                const std::string& key) {
+    auto it = options.find(key);
+    return it == options.end() ? VariantType(NullType()) : VariantType(StringValue(it->second));
+}
+
+Result OptionalLongValue(const std::map& options,
+                                      const std::string& key) {
+    if (options.find(key) == options.end()) {
+        return VariantType(NullType());
+    }
+    PAIMON_ASSIGN_OR_RAISE(int64_t value, OptionsUtils::GetValueFromMap(options, key));
+    return VariantType(value);
+}
+
+Result IsEnabled(const GlobalSystemTableRegistryEntry& entry,
+                       const std::map& catalog_options) {
+    if (entry.name != CatalogOptionsSystemTable::kName) {
+        return true;
+    }
+    return OptionsUtils::GetValueFromMap(catalog_options,
+                                               CatalogOptionsSystemTable::kEnabledOption, false);
+}
+
+struct CatalogTableInfo {
+    std::string database_name;
+    std::string table_name;
+    std::shared_ptr schema;
+};
+
+// Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but
+// propagate all other catalog and schema errors instead of returning incomplete system-table rows.
+Result> LoadAllDataTables(const Catalog& catalog) {
+    std::vector result;
+    PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases());
+    for (const std::string& database : databases) {
+        Result> tables_result = catalog.ListTables(database);
+        if (!tables_result.ok()) {
+            if (tables_result.status().IsNotExist()) {
+                continue;
+            }
+            return tables_result.status();
+        }
+        for (const std::string& table : tables_result.value()) {
+            Identifier identifier(database, table);
+            Result> schema_result = catalog.LoadTableSchema(identifier);
+            if (!schema_result.ok()) {
+                if (schema_result.status().IsNotExist()) {
+                    continue;
+                }
+                return schema_result.status();
+            }
+            std::shared_ptr data_schema =
+                std::dynamic_pointer_cast(schema_result.value());
+            if (!data_schema) {
+                return Status::Invalid("catalog returned a non-data schema for ",
+                                       identifier.ToString());
+            }
+            result.push_back({database, table, std::move(data_schema)});
+        }
+    }
+    return result;
+}
+
+// Aggregated file-level statistics for a table or partition.
+struct FileStats {
+    int64_t record_count = 0;
+    int64_t file_size_in_bytes = 0;
+    int64_t file_count = 0;
+    int64_t last_file_creation_time_millis = 0;
+};
+
+struct AggregatedFileStats {
+    bool has_snapshot = false;
+    std::map by_partition;
+};
+
+// Read the latest snapshot's data files and aggregate statistics.
+Result AggregateFileStats(const std::shared_ptr& fs,
+                                               const std::string& table_path,
+                                               const DataSchema& table_schema) {
+    AggregatedFileStats result;
+
+    SnapshotManager snapshot_manager(fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH);
+    PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot());
+    if (!snapshot) {
+        return result;
+    }
+    result.has_snapshot = true;
+
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(table_schema.Options()));
+
+    auto pool = GetDefaultPool();
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_arrow_schema,
+                           table_schema.GetArrowSchema());
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema,
+                                      arrow::ImportSchema(c_arrow_schema.get()));
+    PAIMON_ASSIGN_OR_RAISE(std::vector external_paths,
+                           core_options.CreateExternalPaths());
+    PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path,
+                           core_options.CreateGlobalIndexExternalPath());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr path_factory,
+        FileStorePathFactory::Create(
+            table_path, arrow_schema, table_schema.PartitionKeys(),
+            core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(),
+            core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
+            external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
+            pool));
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list,
+                           ManifestList::Create(fs, core_options.GetManifestFormat(),
+                                                core_options.GetManifestCompression(), path_factory,
+                                                core_options.GetCache(), pool));
+
+    std::vector manifests;
+    PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(*snapshot, &manifests));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, table_schema.PartitionKeys()));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file,
+                           ManifestFile::Create(fs, core_options.GetManifestFormat(),
+                                                core_options.GetManifestCompression(), path_factory,
+                                                core_options.GetManifestTargetFileSize(), pool,
+                                                core_options, partition_schema));
+
+    std::vector entries;
+    for (const auto& manifest : manifests) {
+        PAIMON_RETURN_NOT_OK(
+            manifest_file->Read(manifest.FileName(), /*filter=*/nullptr, &entries));
+    }
+
+    std::vector merged_entries;
+    PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(entries, &merged_entries));
+
+    for (const auto& entry : merged_entries) {
+        if (!(entry.Kind() == FileKind::Add())) {
+            continue;
+        }
+        const auto& file = entry.File();
+
+        // Convert partition BinaryRow to string representation
+        std::string partition_key;
+        if (entry.Partition().GetFieldCount() > 0) {
+            PAIMON_ASSIGN_OR_RAISE(auto partition_values,
+                                   path_factory->GeneratePartitionVector(entry.Partition()));
+            for (const auto& [key, value] : partition_values) {
+                if (!partition_key.empty()) {
+                    partition_key += "/";
+                }
+                partition_key += key + "=" + value;
+            }
+        }
+
+        auto& stats = result.by_partition[partition_key];
+        stats.record_count += file->row_count;
+        stats.file_size_in_bytes += file->file_size;
+        stats.file_count++;
+        int64_t creation_millis = file->creation_time.GetMillisecond();
+        if (creation_millis > stats.last_file_creation_time_millis) {
+            stats.last_file_creation_time_millis = creation_millis;
+        }
+    }
+
+    return result;
+}
+
+}  // namespace
+
+// =============================================================================
+// GlobalSystemTableLoader
+// =============================================================================
+
+Result GlobalSystemTableLoader::IsSupported(
+    const std::string& table_name, const std::map& catalog_options) {
+    std::string normalized = StringUtils::ToLowerCase(table_name);
+    for (const auto& entry : GlobalSystemTableRegistry()) {
+        if (entry.name == normalized) {
+            return IsEnabled(entry, catalog_options);
+        }
+    }
+    return false;
+}
+
+Result> GlobalSystemTableLoader::Load(
+    const std::string& table_name, const GlobalSystemTableContext& context) {
+    std::string normalized = StringUtils::ToLowerCase(table_name);
+    for (const auto& entry : GlobalSystemTableRegistry()) {
+        if (entry.name == normalized) {
+            PAIMON_ASSIGN_OR_RAISE(bool enabled, IsEnabled(entry, context.catalog_options));
+            if (!enabled) {
+                return Status::NotExist("global system table is disabled: ", table_name);
+            }
+            if (entry.requires_catalog && context.catalog == nullptr) {
+                return Status::NotImplemented("global system table requires catalog context: ",
+                                              table_name);
+            }
+            return entry.factory(context);
+        }
+    }
+    return Status::NotImplemented("unsupported global system table: ", table_name);
+}
+
+Result> GlobalSystemTableLoader::GetSupportedTableNames(
+    const std::map& catalog_options) {
+    std::vector names;
+    names.reserve(GlobalSystemTableRegistry().size());
+    for (const auto& entry : GlobalSystemTableRegistry()) {
+        PAIMON_ASSIGN_OR_RAISE(bool enabled, IsEnabled(entry, catalog_options));
+        if (enabled) {
+            names.push_back(entry.name);
+        }
+    }
+    return names;
+}
+
+// =============================================================================
+// sys.catalog_options
+// =============================================================================
+
+CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context)
+    : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {}
+
+std::string CatalogOptionsSystemTable::Name() const {
+    return kName;
+}
+
+Result> CatalogOptionsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("key", arrow::utf8(), /*nullable=*/false),
+        arrow::field("value", arrow::utf8(), /*nullable=*/false),
+    });
+}
+
+Result> CatalogOptionsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::vector rows;
+    rows.reserve(context_.catalog_options.size());
+    for (const auto& [key, value] : context_.catalog_options) {
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(key));
+        row.SetField(1, StringValue(value));
+        rows.push_back(std::move(row));
+    }
+    return rows;
+}
+
+// =============================================================================
+// sys.all_table_options
+// =============================================================================
+
+AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context)
+    : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {}
+
+std::string AllTableOptionsSystemTable::Name() const {
+    return kName;
+}
+
+Result> AllTableOptionsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("database_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("table_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("key", arrow::utf8(), /*nullable=*/false),
+        arrow::field("value", arrow::utf8(), /*nullable=*/false),
+    });
+}
+
+Result> AllTableOptionsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::vector rows;
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector tables,
+                           LoadAllDataTables(*context_.catalog));
+    for (const CatalogTableInfo& table : tables) {
+        for (const auto& [key, value] : table.schema->Options()) {
+            GenericRow row(schema->num_fields());
+            row.SetField(0, StringValue(table.database_name));
+            row.SetField(1, StringValue(table.table_name));
+            row.SetField(2, StringValue(key));
+            row.SetField(3, StringValue(value));
+            rows.push_back(std::move(row));
+        }
+    }
+    return rows;
+}
+
+// =============================================================================
+// sys.tables
+// =============================================================================
+
+TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context)
+    : InMemorySystemTable("sys/tables"), context_(std::move(context)) {}
+
+std::string TablesSystemTable::Name() const {
+    return kName;
+}
+
+Result> TablesSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("database_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("table_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("table_type", arrow::utf8(), /*nullable=*/false),
+        arrow::field("partitioned", arrow::boolean(), /*nullable=*/false),
+        arrow::field("primary_key", arrow::boolean(), /*nullable=*/false),
+        arrow::field("owner", arrow::utf8(), /*nullable=*/true),
+        arrow::field("created_at", arrow::int64(), /*nullable=*/true),
+        arrow::field("created_by", arrow::utf8(), /*nullable=*/true),
+        arrow::field("updated_at", arrow::int64(), /*nullable=*/true),
+        arrow::field("updated_by", arrow::utf8(), /*nullable=*/true),
+        arrow::field("record_count", arrow::int64(), /*nullable=*/true),
+        arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true),
+        arrow::field("file_count", arrow::int64(), /*nullable=*/true),
+        arrow::field("last_file_creation_time", arrow::int64(), /*nullable=*/true),
+    });
+}
+
+Result> TablesSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::vector rows;
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector tables,
+                           LoadAllDataTables(*context_.catalog));
+    for (const CatalogTableInfo& table : tables) {
+        const std::shared_ptr& data_schema = table.schema;
+
+        const auto& opts = data_schema->Options();
+        auto table_type = opts.find("type");
+        const std::string table_type_str = table_type == opts.end() ? "table" : table_type->second;
+
+        bool partitioned = !data_schema->PartitionKeys().empty();
+
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(table.database_name));
+        row.SetField(1, StringValue(table.table_name));
+        row.SetField(2, StringValue(table_type_str));
+        row.SetField(3, partitioned);
+        row.SetField(4, !data_schema->PrimaryKeys().empty());
+        row.SetField(5, OptionalStringValue(opts, "owner"));
+        PAIMON_ASSIGN_OR_RAISE(VariantType created_at, OptionalLongValue(opts, "createdAt"));
+        row.SetField(6, std::move(created_at));
+        row.SetField(7, OptionalStringValue(opts, "createdBy"));
+        PAIMON_ASSIGN_OR_RAISE(VariantType updated_at, OptionalLongValue(opts, "updatedAt"));
+        row.SetField(8, std::move(updated_at));
+        row.SetField(9, OptionalStringValue(opts, "updatedBy"));
+
+        // Match Java CatalogUtils::toTableAndSnapshots when version management is unsupported.
+        // The C++ Catalog API currently has no version-management capability, so leave snapshot
+        // statistics null instead of deriving different live-file semantics from manifests.
+        row.SetField(10, NullType());
+        row.SetField(11, NullType());
+        row.SetField(12, NullType());
+        row.SetField(13, NullType());
+
+        rows.push_back(std::move(row));
+    }
+    return rows;
+}
+
+// =============================================================================
+// sys.partitions
+// =============================================================================
+
+PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context)
+    : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {}
+
+std::string PartitionsSystemTable::Name() const {
+    return kName;
+}
+
+Result> PartitionsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("database_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("table_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("partition_name", arrow::utf8(), /*nullable=*/true),
+        arrow::field("record_count", arrow::int64(), /*nullable=*/true),
+        arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true),
+        arrow::field("file_count", arrow::int64(), /*nullable=*/true),
+        arrow::field("last_file_creation_time", arrow::int64(), /*nullable=*/true),
+        arrow::field("done", arrow::boolean(), /*nullable=*/false),
+    });
+}
+
+Result> PartitionsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::vector rows;
+
+    PAIMON_ASSIGN_OR_RAISE(std::vector tables,
+                           LoadAllDataTables(*context_.catalog));
+    for (const CatalogTableInfo& table : tables) {
+        const std::shared_ptr& data_schema = table.schema;
+        Identifier id(table.database_name, table.table_name);
+
+        // Only emit rows for partitioned tables
+        if (data_schema->PartitionKeys().empty()) {
+            continue;
+        }
+
+        // Match Java's toAllPartitions by ignoring only concurrent table deletion. All other
+        // metadata and I/O errors must fail the query.
+        Result table_path_result = context_.catalog->GetTableLocation(id);
+        if (!table_path_result.ok()) {
+            if (table_path_result.status().IsNotExist()) {
+                continue;
+            }
+            return table_path_result.status();
+        }
+
+        Result file_stats_result =
+            AggregateFileStats(context_.fs, table_path_result.value(), *data_schema);
+        if (!file_stats_result.ok()) {
+            if (file_stats_result.status().IsNotExist()) {
+                continue;
+            }
+            return file_stats_result.status();
+        }
+
+        auto& stats_map = file_stats_result.value().by_partition;
+        for (const auto& [partition_key, stats] : stats_map) {
+            if (stats.file_count == 0) {
+                continue;
+            }
+            GenericRow row(schema->num_fields());
+            row.SetField(0, StringValue(table.database_name));
+            row.SetField(1, StringValue(table.table_name));
+            row.SetField(2, partition_key.empty() ? VariantType(NullType())
+                                                  : VariantType(StringValue(partition_key)));
+            row.SetField(3, VariantType(stats.record_count));
+            row.SetField(4, VariantType(stats.file_size_in_bytes));
+            row.SetField(5, VariantType(stats.file_count));
+            row.SetField(6, stats.last_file_creation_time_millis > 0
+                                ? VariantType(stats.last_file_creation_time_millis)
+                                : VariantType(NullType()));
+            // File-system catalog partitions are not explicitly marked as done.
+            row.SetField(7, false);
+            rows.push_back(std::move(row));
+        }
+    }
+    return rows;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h
new file mode 100644
index 00000000..3d693626
--- /dev/null
+++ b/src/paimon/core/table/system/global_system_tables.h
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/table/system/in_memory_system_table.h"
+
+namespace paimon {
+class Catalog;
+class FileSystem;
+
+/// Context passed to global system table constructors, providing catalog-level
+/// access for enumerating databases, tables, and reading metadata.
+struct GlobalSystemTableContext {
+    const Catalog* catalog = nullptr;  // non-owning pointer
+    std::shared_ptr fs;
+    std::string warehouse;
+    std::map catalog_options;
+};
+
+/// System table for `sys.catalog_options`, exposing catalog-level configuration
+/// as key/value rows.
+class CatalogOptionsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "catalog_options";
+    static constexpr const char* kEnabledOption = "catalog-options-table.enabled";
+
+    explicit CatalogOptionsSystemTable(GlobalSystemTableContext context);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    GlobalSystemTableContext context_;
+};
+
+/// System table for `sys.all_table_options`, exposing all table options across
+/// all databases as (database_name, table_name, key, value) rows.
+class AllTableOptionsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "all_table_options";
+
+    explicit AllTableOptionsSystemTable(GlobalSystemTableContext context);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    GlobalSystemTableContext context_;
+};
+
+/// System table for `sys.tables`, exposing metadata for all tables across all
+/// databases including record counts and file statistics.
+class TablesSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "tables";
+
+    explicit TablesSystemTable(GlobalSystemTableContext context);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    GlobalSystemTableContext context_;
+};
+
+/// System table for `sys.partitions`, exposing partition-level file statistics
+/// for all tables across all databases.
+class PartitionsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "partitions";
+
+    explicit PartitionsSystemTable(GlobalSystemTableContext context);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    GlobalSystemTableContext context_;
+};
+
+/// Loader for global system tables under the `sys` database.
+///
+/// Maintains its own registry with a factory signature that receives a
+/// GlobalSystemTableContext instead of a per-table TableSchema.
+class GlobalSystemTableLoader {
+ public:
+    static Result IsSupported(const std::string& table_name,
+                                    const std::map& catalog_options);
+
+    static Result> Load(const std::string& table_name,
+                                                     const GlobalSystemTableContext& context);
+
+    static Result> GetSupportedTableNames(
+        const std::map& catalog_options);
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp
new file mode 100644
index 00000000..91cb9eec
--- /dev/null
+++ b/src/paimon/core/table/system/in_memory_system_table.cpp
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/system/in_memory_system_table.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/core/io/generic_row_to_arrow_array_converter.h"
+#include "paimon/core/table/system/system_table_scan.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/read_context.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+
+namespace paimon {
+namespace {
+
+class InMemorySystemTableBatchReader : public BatchReader {
+ public:
+    InMemorySystemTableBatchReader(std::shared_ptr table,
+                                   const std::shared_ptr& pool)
+        : table_(std::move(table)), arrow_pool_(GetArrowPool(pool)) {}
+
+    Result NextBatch() override {
+        if (emitted_) {
+            return BatchReader::MakeEofBatch();
+        }
+        emitted_ = true;
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, table_->ArrowSchema());
+        PAIMON_ASSIGN_OR_RAISE(std::vector rows, table_->BuildRows());
+        if (rows.empty()) {
+            return BatchReader::MakeEofBatch();
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr converter,
+                               GenericRowToArrowArrayConverter::Create(schema, arrow_pool_.get()));
+        return converter->NextBatch(rows);
+    }
+
+    std::shared_ptr GetReaderMetrics() const override {
+        return std::make_shared();
+    }
+
+    void Close() override {
+        emitted_ = true;
+    }
+
+ private:
+    std::shared_ptr table_;
+    std::unique_ptr arrow_pool_;
+    bool emitted_ = false;
+};
+
+class InMemorySystemTableRead : public TableRead {
+ public:
+    InMemorySystemTableRead(std::shared_ptr table,
+                            const std::shared_ptr& memory_pool)
+        : TableRead(memory_pool), table_(std::move(table)) {}
+
+    Result> CreateReader(
+        const std::vector>& splits) override {
+        if (splits.size() != 1) {
+            return Status::Invalid(table_->Name(), " system table expects a single split");
+        }
+        for (const auto& split : splits) {
+            if (!std::dynamic_pointer_cast(split)) {
+                return Status::Invalid("unsupported split for ", table_->Name(), " system table");
+            }
+        }
+        return std::make_unique(table_, GetMemoryPool());
+    }
+
+    Result> CreateReader(
+        const std::shared_ptr& split) override {
+        std::vector> splits = {split};
+        return CreateReader(splits);
+    }
+
+ private:
+    std::shared_ptr table_;
+};
+
+}  // namespace
+
+InMemorySystemTable::InMemorySystemTable(std::string table_path)
+    : table_path_(std::move(table_path)) {}
+
+Result> InMemorySystemTable::NewScan(
+    const std::shared_ptr& /*context*/) const {
+    return std::make_unique(table_path_);
+}
+
+Result> InMemorySystemTable::NewRead(
+    const std::shared_ptr& context) const {
+    return std::make_unique(
+        std::static_pointer_cast(shared_from_this()),
+        context->GetMemoryPool());
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/in_memory_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h
new file mode 100644
index 00000000..f9fcebdb
--- /dev/null
+++ b/src/paimon/core/table/system/in_memory_system_table.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/generic_row.h"
+#include "paimon/core/table/system/system_table.h"
+
+namespace paimon {
+
+/// Base class for system tables whose result can be materialized as a single in-memory
+/// RecordBatch.
+///
+/// It provides the common singleton split scan and one-shot batch reader.
+class InMemorySystemTable : public SystemTable {
+ public:
+    explicit InMemorySystemTable(std::string table_path);
+
+    Result> NewScan(
+        const std::shared_ptr& context) const override;
+    Result> NewRead(
+        const std::shared_ptr& context) const override;
+    virtual Result> BuildRows() const = 0;
+
+ protected:
+    const std::string& TablePath() const {
+        return table_path_;
+    }
+
+ private:
+    std::string table_path_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp
new file mode 100644
index 00000000..4467b7b9
--- /dev/null
+++ b/src/paimon/core/table/system/metadata_system_tables.cpp
@@ -0,0 +1,969 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/system/metadata_system_tables.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/data/data_define.h"
+#include "paimon/common/data/generic_row.h"
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/internal_row_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/rapidjson_util.h"
+#include "paimon/core/casting/cast_executor_factory.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_entry.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/manifest_entry.h"
+#include "paimon/core/manifest/manifest_file.h"
+#include "paimon/core/manifest/manifest_file_meta.h"
+#include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/stats/simple_stats_evolution.h"
+#include "paimon/core/tag/tag.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/consumer_manager.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/core/utils/tag_manager.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
+#include "rapidjson/document.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+
+namespace paimon {
+namespace {
+
+constexpr int32_t kMaxPartitionStatsLength = 255;
+
+template 
+Result JsonString(const T& value) {
+    rapidjson::Document document;
+    auto json_value = RapidJsonUtil::SerializeValue(value, &document.GetAllocator());
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer writer(buffer);
+    if (!json_value.Accept(writer)) {
+        return Status::Invalid("failed to serialize metadata system table value");
+    }
+    return std::string(buffer.GetString(), buffer.GetSize());
+}
+
+Result LocalDateTimePartsToTimestampMillis(const std::vector& parts) {
+    if (parts.size() < 6) {
+        return Status::Invalid("tag create time requires at least 6 date-time fields");
+    }
+
+    int64_t year = parts[0];
+    int64_t month = parts[1];
+    int64_t day = parts[2];
+    int64_t hour = parts[3];
+    int64_t minute = parts[4];
+    int64_t second = parts[5];
+    int64_t nanos = parts.size() > 6 ? parts[6] : 0;
+    auto is_leap_year = [](int64_t value) {
+        return value % 4 == 0 && (value % 100 != 0 || value % 400 == 0);
+    };
+    int64_t days_in_month[] = {31, is_leap_year(year) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30,
+                               31};
+    if (month < 1 || month > 12 || day < 1 || day > days_in_month[month - 1] || hour < 0 ||
+        hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 59 || nanos < 0 ||
+        nanos > 999999999) {
+        return Status::Invalid("invalid tag create time fields");
+    }
+
+    year -= month <= 2 ? 1 : 0;
+    int64_t era = (year >= 0 ? year : year - 399) / 400;
+    auto year_of_era = static_cast(year - era * 400);
+    auto month_prime = static_cast(month + (month > 2 ? -3 : 9));
+    uint32_t day_of_year = (153 * month_prime + 2) / 5 + static_cast(day) - 1;
+    uint32_t day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
+    int64_t epoch_day = era * 146097 + static_cast(day_of_era) - 719468;
+    return epoch_day * DateTimeUtils::MILLIS_PER_DAY + hour * 3600000 + minute * 60000 +
+           second * 1000 + nanos / 1000000;
+}
+
+Result> OptionalLocalDateTimePartsToTimestampMillis(
+    const std::optional>& parts) {
+    if (!parts) {
+        return std::optional();
+    }
+    PAIMON_ASSIGN_OR_RAISE(int64_t timestamp_millis,
+                           LocalDateTimePartsToTimestampMillis(parts.value()));
+    return std::optional(timestamp_millis);
+}
+
+std::optional OptionalDoubleToString(const std::optional& value) {
+    if (!value) {
+        return std::optional();
+    }
+    return std::to_string(value.value());
+}
+
+VariantType OptionalInt64Value(const std::optional& value) {
+    if (!value) {
+        return NullType();
+    }
+    return value.value();
+}
+
+VariantType StringValue(const std::string& value) {
+    return BinaryString::FromString(value, GetDefaultPool().get());
+}
+
+VariantType OptionalStringValue(const std::optional& value) {
+    if (!value) {
+        return NullType();
+    }
+    return StringValue(value.value());
+}
+
+VariantType TimestampMillisValue(int64_t value) {
+    return Timestamp::FromEpochMillis(value);
+}
+
+Result LocalTimestampMillisValue(int64_t epoch_millis) {
+    PAIMON_ASSIGN_OR_RAISE(
+        Timestamp local_timestamp,
+        DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(epoch_millis)));
+    return TimestampMillisValue(local_timestamp.GetMillisecond());
+}
+
+VariantType OptionalTimestampMillisValue(const std::optional& value) {
+    if (!value) {
+        return NullType();
+    }
+    return TimestampMillisValue(value.value());
+}
+
+MetadataSystemTableContext CreateMetadataContext(std::shared_ptr fs,
+                                                 std::string table_path, std::string branch) {
+    return {
+        std::move(fs), std::move(table_path), BranchManager::NormalizeBranch(branch), nullptr, {},
+    };
+}
+
+MetadataSystemTableContext CreateMetadataContext(std::shared_ptr fs,
+                                                 std::string table_path, std::string branch,
+                                                 std::shared_ptr table_schema,
+                                                 std::map options) {
+    return {
+        std::move(fs),           std::move(table_path), BranchManager::NormalizeBranch(branch),
+        std::move(table_schema), std::move(options),
+    };
+}
+
+Result CreateCoreOptions(const MetadataSystemTableContext& context) {
+    return CoreOptions::FromMap(context.options, context.fs);
+}
+
+Result> CreatePathFactory(
+    const MetadataSystemTableContext& context, const CoreOptions& core_options,
+    const std::shared_ptr& pool) {
+    std::shared_ptr arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context.table_schema->Fields());
+    PAIMON_ASSIGN_OR_RAISE(std::vector external_paths,
+                           core_options.CreateExternalPaths());
+    PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path,
+                           core_options.CreateGlobalIndexExternalPath());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr path_factory,
+        FileStorePathFactory::Create(
+            context.table_path, arrow_schema, context.table_schema->PartitionKeys(),
+            core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(),
+            core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(),
+            external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(),
+            pool));
+    return path_factory;
+}
+
+Result> LatestSnapshot(const MetadataSystemTableContext& context) {
+    SnapshotManager snapshot_manager(context.fs, context.table_path, context.branch);
+    return snapshot_manager.LatestSnapshot();
+}
+
+Result> ReadDataManifests(
+    const MetadataSystemTableContext& context, const Snapshot& snapshot,
+    const std::shared_ptr& path_factory, const CoreOptions& core_options,
+    const std::shared_ptr& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list,
+                           ManifestList::Create(context.fs, core_options.GetManifestFormat(),
+                                                core_options.GetManifestCompression(), path_factory,
+                                                core_options.GetCache(), pool));
+    std::vector manifests;
+    // TODO(suxiaogang223): Align Java ReadAllManifests semantics by including changelog
+    // manifests. ReadAllManifests currently delegates to ReadChangelogManifests, which returns
+    // NotImplemented when a snapshot has a changelog manifest list.
+    PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(snapshot, &manifests));
+    return manifests;
+}
+
+Result> CreateManifestFile(
+    const MetadataSystemTableContext& context,
+    const std::shared_ptr& path_factory, const CoreOptions& core_options,
+    const std::shared_ptr& pool) {
+    std::shared_ptr arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context.table_schema->Fields());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, context.table_schema->PartitionKeys()));
+    return ManifestFile::Create(context.fs, core_options.GetManifestFormat(),
+                                core_options.GetManifestCompression(), path_factory,
+                                core_options.GetManifestTargetFileSize(), pool, core_options,
+                                partition_schema);
+}
+
+Result> ReadLatestManifestEntries(
+    const MetadataSystemTableContext& context,
+    const std::shared_ptr& path_factory, const CoreOptions& core_options,
+    const std::shared_ptr& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, LatestSnapshot(context));
+    if (!snapshot) {
+        return std::vector();
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector manifests,
+        ReadDataManifests(context, snapshot.value(), path_factory, core_options, pool));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file,
+                           CreateManifestFile(context, path_factory, core_options, pool));
+    std::vector entries;
+    for (const auto& manifest : manifests) {
+        PAIMON_RETURN_NOT_OK(
+            manifest_file->Read(manifest.FileName(), /*filter=*/nullptr, &entries));
+    }
+    return entries;
+}
+
+Result> ReadLatestDataFiles(
+    const MetadataSystemTableContext& context,
+    const std::shared_ptr& path_factory, const CoreOptions& core_options,
+    const std::shared_ptr& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector entries,
+                           ReadLatestManifestEntries(context, path_factory, core_options, pool));
+    std::vector merged_entries;
+    PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(entries, &merged_entries));
+    return merged_entries;
+}
+
+Result> OptionalPartitionString(
+    const BinaryRow& row, const std::shared_ptr& partition_schema) {
+    if (row.GetFieldCount() <= 0) {
+        return std::optional();
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string value,
+                           BinaryRowPartitionComputer::PartToSimpleString(
+                               partition_schema, row, ",", kMaxPartitionStatsLength,
+                               /*legacy_partition_name_enabled=*/false));
+    return std::optional(fmt::format("{{{}}}", value));
+}
+
+Result OptionalPartitionStringValue(
+    const BinaryRow& row, const std::shared_ptr& partition_schema) {
+    PAIMON_ASSIGN_OR_RAISE(std::optional value,
+                           OptionalPartitionString(row, partition_schema));
+    return OptionalStringValue(value);
+}
+
+Result FilePath(const std::shared_ptr& path_factory,
+                             const ManifestEntry& entry, const DataFileMeta& file) {
+    if (file.external_path) {
+        return file.external_path.value();
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string bucket_path,
+                           path_factory->BucketPath(entry.Partition(), entry.Bucket()));
+    return PathUtil::JoinPath(bucket_path, file.file_name);
+}
+
+Result FieldValueString(const DataField& field, const VariantType& value) {
+    PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
+                           FieldTypeUtils::ConvertToFieldType(field.Type()->id()));
+    std::shared_ptr cast_executor =
+        CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(field_type,
+                                                                       FieldType::STRING);
+    if (!cast_executor) {
+        return DataDefine::VariantValueToString(value);
+    }
+    PAIMON_ASSIGN_OR_RAISE(Literal literal,
+                           DataDefine::VariantValueToLiteral(value, field.Type()->id()));
+    PAIMON_ASSIGN_OR_RAISE(Literal string_literal, cast_executor->Cast(literal, arrow::utf8()));
+    return string_literal.GetValue();
+}
+
+Result> RowValueStrings(const std::vector& fields,
+                                                 const InternalRow& row) {
+    std::shared_ptr schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+    PAIMON_ASSIGN_OR_RAISE(std::vector getters,
+                           InternalRowUtils::CreateFieldGetters(schema, /*use_view=*/false));
+    std::vector values;
+    int32_t length = std::min(static_cast(fields.size()), row.GetFieldCount());
+    values.reserve(length);
+    for (int32_t i = 0; i < length; ++i) {
+        std::string value = "null";
+        if (!row.IsNullAt(i)) {
+            VariantType field_value = getters[i](row);
+            PAIMON_ASSIGN_OR_RAISE(value, FieldValueString(fields[i], field_value));
+        }
+        values.push_back(std::move(value));
+    }
+    return values;
+}
+
+struct StatsStringOverrides {
+    std::map values;
+    std::map null_counts;
+};
+
+Result OmittedPartitionStats(const std::shared_ptr& table_schema,
+                                                   const DataFileMeta& file,
+                                                   const BinaryRow& partition, int64_t row_count) {
+    StatsStringOverrides overrides;
+    if (!file.write_cols) {
+        return overrides;
+    }
+
+    std::vector partition_fields;
+    partition_fields.reserve(table_schema->PartitionKeys().size());
+    for (const auto& partition_key : table_schema->PartitionKeys()) {
+        PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(partition_key));
+        partition_fields.push_back(std::move(field));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector partition_values,
+                           RowValueStrings(partition_fields, partition));
+    if (partition_fields.size() != partition_values.size()) {
+        return Status::Invalid(
+            fmt::format("partition field count {} does not match partition value count {}",
+                        partition_fields.size(), partition_values.size()));
+    }
+    for (size_t i = 0; i < partition_fields.size(); ++i) {
+        const std::string& field_name = partition_fields[i].Name();
+        if (std::find(file.write_cols->begin(), file.write_cols->end(), field_name) !=
+            file.write_cols->end()) {
+            continue;
+        }
+        overrides.values.emplace(field_name, std::move(partition_values[i]));
+        overrides.null_counts.emplace(field_name,
+                                      partition.IsNullAt(i) ? std::to_string(row_count) : "0");
+    }
+    return overrides;
+}
+
+Result RowValuesString(const std::vector& fields, const InternalRow& row,
+                                    std::string_view left, std::string_view right) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row));
+    return fmt::format("{}{}{}", left, fmt::join(values, ", "), right);
+}
+
+Result> OptionalRowValuesString(const std::vector& fields,
+                                                           const InternalRow& row,
+                                                           std::string_view left,
+                                                           std::string_view right) {
+    if (row.GetFieldCount() <= 0) {
+        return std::optional();
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::string value, RowValuesString(fields, row, left, right));
+    return std::optional(value);
+}
+
+Result FieldsValueMapString(const std::vector& fields,
+                                         const InternalRow& row,
+                                         const std::map& overrides) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row));
+    std::vector> field_values;
+    size_t length = std::min(fields.size(), values.size());
+    field_values.reserve(length);
+    for (size_t i = 0; i < length; ++i) {
+        auto iter = overrides.find(fields[i].Name());
+        field_values.emplace_back(fields[i].Name(),
+                                  iter == overrides.end() ? std::move(values[i]) : iter->second);
+    }
+    std::sort(field_values.begin(), field_values.end(),
+              [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; });
+
+    std::vector entries;
+    entries.reserve(field_values.size());
+    for (const auto& [name, value] : field_values) {
+        entries.emplace_back(fmt::format("{}={}", name, value));
+    }
+    return fmt::format("{{{}}}", fmt::join(entries, ", "));
+}
+
+Result NullValueCountsString(const std::vector& fields,
+                                          const InternalArray& null_counts,
+                                          const std::map& overrides) {
+    std::vector> field_values;
+    int32_t length = std::min(static_cast(fields.size()), null_counts.Size());
+    field_values.reserve(length);
+    for (int32_t i = 0; i < length; ++i) {
+        auto iter = overrides.find(fields[i].Name());
+        std::string value =
+            iter == overrides.end()
+                ? (null_counts.IsNullAt(i) ? "null" : std::to_string(null_counts.GetLong(i)))
+                : iter->second;
+        field_values.emplace_back(fields[i].Name(), std::move(value));
+    }
+    std::sort(field_values.begin(), field_values.end(),
+              [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; });
+
+    std::vector entries;
+    entries.reserve(field_values.size());
+    for (const auto& [name, value] : field_values) {
+        entries.emplace_back(fmt::format("{}={}", name, value));
+    }
+    return fmt::format("{{{}}}", fmt::join(entries, ", "));
+}
+
+Result> LoadDataSchema(const MetadataSystemTableContext& context,
+                                                    int64_t schema_id) {
+    if (schema_id == context.table_schema->Id()) {
+        return context.table_schema;
+    }
+    SchemaManager schema_manager(context.fs, context.table_path, context.branch);
+    return schema_manager.ReadSchema(schema_id);
+}
+
+Result> ProjectWriteFields(const std::shared_ptr& data_schema,
+                                                  const DataFileMeta& file) {
+    if (!file.write_cols) {
+        return data_schema->Fields();
+    }
+
+    std::vector fields;
+    fields.reserve(file.write_cols->size());
+    for (const auto& write_col : file.write_cols.value()) {
+        if (SpecialFields::IsSystemField(write_col)) {
+            continue;
+        }
+        PAIMON_ASSIGN_OR_RAISE(DataField field, data_schema->GetField(write_col));
+        fields.push_back(std::move(field));
+    }
+    return fields;
+}
+
+Result> KeyFieldsForFilesTable(
+    const std::shared_ptr& data_schema) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector key_fields,
+                           data_schema->TrimmedPrimaryKeyFields());
+    // Java FilesTable falls back to logicalRowType when logicalTrimmedPrimaryKeysType is empty.
+    if (key_fields.empty()) {
+        return data_schema->Fields();
+    }
+    return key_fields;
+}
+
+Result> WriteColsValue(
+    const std::optional>& write_cols,
+    const std::shared_ptr& pool) {
+    if (!write_cols) {
+        return std::shared_ptr();
+    }
+    return std::make_shared(
+        InternalRowUtils::ToNotNullStringArrayData(write_cols.value(), pool));
+}
+
+}  // namespace
+
+OptionsSystemTable::OptionsSystemTable(std::string table_path,
+                                       std::shared_ptr table_schema)
+    : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {}
+
+std::string OptionsSystemTable::Name() const {
+    return kName;
+}
+
+Result> OptionsSystemTable::ArrowSchema() const {
+    return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false),
+                          arrow::field("value", arrow::utf8(), /*nullable=*/false)});
+}
+
+Result> OptionsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::vector rows;
+    rows.reserve(table_schema_->Options().size());
+    for (const auto& [key, value] : table_schema_->Options()) {
+        GenericRow row(schema->num_fields());
+        row.SetField(0, std::string_view(key));
+        row.SetField(1, std::string_view(value));
+        rows.push_back(std::move(row));
+    }
+    return rows;
+}
+
+SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path,
+                                           std::string branch)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {}
+
+std::string SnapshotsSystemTable::Name() const {
+    return kName;
+}
+
+Result> SnapshotsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("schema_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("commit_user", arrow::utf8(), /*nullable=*/false),
+        arrow::field("commit_identifier", arrow::int64(), /*nullable=*/false),
+        arrow::field("commit_kind", arrow::utf8(), /*nullable=*/false),
+        arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/false),
+        arrow::field("base_manifest_list", arrow::utf8(), /*nullable=*/false),
+        arrow::field("delta_manifest_list", arrow::utf8(), /*nullable=*/false),
+        arrow::field("changelog_manifest_list", arrow::utf8(), /*nullable=*/true),
+        arrow::field("total_record_count", arrow::int64(), /*nullable=*/false),
+        arrow::field("delta_record_count", arrow::int64(), /*nullable=*/false),
+        arrow::field("changelog_record_count", arrow::int64(), /*nullable=*/true),
+        arrow::field("watermark", arrow::int64(), /*nullable=*/true),
+        arrow::field("next_row_id", arrow::int64(), /*nullable=*/true),
+    });
+}
+
+Result> SnapshotsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    SnapshotManager snapshot_manager(context_.fs, context_.table_path, context_.branch);
+    PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, snapshot_manager.GetAllSnapshots());
+    std::sort(snapshots.begin(), snapshots.end(),
+              [](const Snapshot& lhs, const Snapshot& rhs) { return lhs.Id() < rhs.Id(); });
+    std::vector rows;
+    rows.reserve(snapshots.size());
+
+    for (const auto& snapshot : snapshots) {
+        GenericRow row(schema->num_fields());
+        row.SetField(0, snapshot.Id());
+        row.SetField(1, snapshot.SchemaId());
+        row.SetField(2, StringValue(snapshot.CommitUser()));
+        row.SetField(3, snapshot.CommitIdentifier());
+        row.SetField(4, StringValue(Snapshot::CommitKind::ToString(snapshot.GetCommitKind())));
+        PAIMON_ASSIGN_OR_RAISE(VariantType commit_time,
+                               LocalTimestampMillisValue(snapshot.TimeMillis()));
+        row.SetField(5, commit_time);
+        row.SetField(6, StringValue(snapshot.BaseManifestList()));
+        row.SetField(7, StringValue(snapshot.DeltaManifestList()));
+        row.SetField(8, OptionalStringValue(snapshot.ChangelogManifestList()));
+        row.SetField(9, snapshot.TotalRecordCount());
+        row.SetField(10, snapshot.DeltaRecordCount());
+        row.SetField(11, OptionalInt64Value(snapshot.ChangelogRecordCount()));
+        row.SetField(12, OptionalInt64Value(snapshot.Watermark()));
+        row.SetField(13, OptionalInt64Value(snapshot.NextRowId()));
+        rows.push_back(std::move(row));
+    }
+
+    return rows;
+}
+
+SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path,
+                                       std::string branch)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {}
+
+std::string SchemasSystemTable::Name() const {
+    return kName;
+}
+
+Result> SchemasSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("schema_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("fields", arrow::utf8(), /*nullable=*/false),
+        arrow::field("partition_keys", arrow::utf8(), /*nullable=*/false),
+        arrow::field("primary_keys", arrow::utf8(), /*nullable=*/false),
+        arrow::field("options", arrow::utf8(), /*nullable=*/false),
+        arrow::field("comment", arrow::utf8(), /*nullable=*/true),
+        arrow::field("update_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/false),
+    });
+}
+
+Result> SchemasSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    SchemaManager schema_manager(context_.fs, context_.table_path, context_.branch);
+    PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager.ListAllIds());
+    std::sort(schema_ids.begin(), schema_ids.end());
+    std::vector rows;
+    rows.reserve(schema_ids.size());
+
+    for (int64_t id : schema_ids) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema,
+                               schema_manager.ReadSchema(id));
+        PAIMON_ASSIGN_OR_RAISE(std::string fields_json, JsonString(table_schema->Fields()));
+        PAIMON_ASSIGN_OR_RAISE(std::string partition_keys_json,
+                               JsonString(table_schema->PartitionKeys()));
+        PAIMON_ASSIGN_OR_RAISE(std::string primary_keys_json,
+                               JsonString(table_schema->PrimaryKeys()));
+        PAIMON_ASSIGN_OR_RAISE(std::string options_json, JsonString(table_schema->Options()));
+
+        GenericRow row(schema->num_fields());
+        row.SetField(0, table_schema->Id());
+        row.SetField(1, StringValue(fields_json));
+        row.SetField(2, StringValue(partition_keys_json));
+        row.SetField(3, StringValue(primary_keys_json));
+        row.SetField(4, StringValue(options_json));
+        row.SetField(5, OptionalStringValue(table_schema->Comment()));
+        PAIMON_ASSIGN_OR_RAISE(VariantType update_time,
+                               LocalTimestampMillisValue(table_schema->TimeMillis()));
+        row.SetField(6, update_time);
+        rows.push_back(std::move(row));
+    }
+
+    return rows;
+}
+
+TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path,
+                                 std::string branch)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {}
+
+std::string TagsSystemTable::Name() const {
+    return kName;
+}
+
+Result> TagsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("tag_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("schema_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/false),
+        arrow::field("record_count", arrow::int64(), /*nullable=*/false),
+        arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/true),
+        arrow::field("time_retained", arrow::utf8(), /*nullable=*/true),
+    });
+}
+
+Result> TagsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    TagManager tag_manager(context_.fs, context_.table_path, context_.branch);
+    PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames());
+    std::vector rows;
+    rows.reserve(tag_names.size());
+
+    for (const auto& name : tag_names) {
+        PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(name));
+        PAIMON_ASSIGN_OR_RAISE(std::optional tag_create_time,
+                               OptionalLocalDateTimePartsToTimestampMillis(tag.TagCreateTime()));
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(name));
+        row.SetField(1, tag.Id());
+        row.SetField(2, tag.SchemaId());
+        PAIMON_ASSIGN_OR_RAISE(VariantType commit_time,
+                               LocalTimestampMillisValue(tag.TimeMillis()));
+        row.SetField(3, commit_time);
+        row.SetField(4, tag.TotalRecordCount());
+        row.SetField(5, OptionalTimestampMillisValue(tag_create_time));
+        row.SetField(6, OptionalStringValue(OptionalDoubleToString(tag.TagTimeRetained())));
+        rows.push_back(std::move(row));
+    }
+
+    return rows;
+}
+
+BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path,
+                                         std::string branch)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {}
+
+std::string BranchesSystemTable::Name() const {
+    return kName;
+}
+
+Result> BranchesSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("branch_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/false),
+    });
+}
+
+Result> BranchesSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    PAIMON_ASSIGN_OR_RAISE(std::vector branches,
+                           BranchManager::ListBranches(context_.fs, context_.table_path));
+    std::vector rows;
+    rows.reserve(branches.size());
+
+    for (const auto& name : branches) {
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr branch_status,
+            context_.fs->GetFileStatus(BranchManager::BranchPath(context_.table_path, name)));
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(name));
+        PAIMON_ASSIGN_OR_RAISE(VariantType create_time,
+                               LocalTimestampMillisValue(branch_status->GetModificationTime()));
+        row.SetField(1, create_time);
+        rows.push_back(std::move(row));
+    }
+
+    return rows;
+}
+
+ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path,
+                                           std::string branch)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {}
+
+std::string ConsumersSystemTable::Name() const {
+    return kName;
+}
+
+Result> ConsumersSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false),
+        arrow::field("next_snapshot_id", arrow::int64(), /*nullable=*/false),
+    });
+}
+
+Result> ConsumersSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    ConsumerManager consumer_manager(context_.fs, context_.table_path, context_.branch);
+    PAIMON_ASSIGN_OR_RAISE(auto consumers, consumer_manager.Consumers());
+    std::vector rows;
+    rows.reserve(consumers.size());
+
+    for (const auto& [id, snapshot_id] : consumers) {
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(id));
+        row.SetField(1, snapshot_id);
+        rows.push_back(std::move(row));
+    }
+
+    return rows;
+}
+
+ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std::string table_path,
+                                           std::string branch,
+                                           std::shared_ptr table_schema,
+                                           std::map options)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch),
+                                     std::move(table_schema), std::move(options))) {}
+
+std::string ManifestsSystemTable::Name() const {
+    return kName;
+}
+
+Result> ManifestsSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("file_name", arrow::utf8(), /*nullable=*/false),
+        arrow::field("file_size", arrow::int64(), /*nullable=*/false),
+        arrow::field("num_added_files", arrow::int64(), /*nullable=*/false),
+        arrow::field("num_deleted_files", arrow::int64(), /*nullable=*/false),
+        arrow::field("schema_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("min_partition_stats", arrow::utf8(), /*nullable=*/true),
+        arrow::field("max_partition_stats", arrow::utf8(), /*nullable=*/true),
+        arrow::field("min_row_id", arrow::int64(), /*nullable=*/true),
+        arrow::field("max_row_id", arrow::int64(), /*nullable=*/true),
+    });
+}
+
+Result> ManifestsSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, LatestSnapshot(context_));
+    if (!snapshot) {
+        return std::vector();
+    }
+
+    std::shared_ptr pool = GetDefaultPool();
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory,
+                           CreatePathFactory(context_, core_options, pool));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::vector manifests,
+        ReadDataManifests(context_, snapshot.value(), path_factory, core_options, pool));
+    std::shared_ptr arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context_.table_schema->Fields());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, context_.table_schema->PartitionKeys()));
+
+    std::vector rows;
+    rows.reserve(manifests.size());
+    for (const auto& manifest : manifests) {
+        GenericRow row(schema->num_fields());
+        row.SetField(0, StringValue(manifest.FileName()));
+        row.SetField(1, manifest.FileSize());
+        row.SetField(2, manifest.NumAddedFiles());
+        row.SetField(3, manifest.NumDeletedFiles());
+        row.SetField(4, manifest.SchemaId());
+        PAIMON_ASSIGN_OR_RAISE(
+            VariantType min_partition,
+            OptionalPartitionStringValue(manifest.PartitionStats().MinValues(), partition_schema));
+        PAIMON_ASSIGN_OR_RAISE(
+            VariantType max_partition,
+            OptionalPartitionStringValue(manifest.PartitionStats().MaxValues(), partition_schema));
+        row.SetField(5, min_partition);
+        row.SetField(6, max_partition);
+        row.SetField(7, OptionalInt64Value(manifest.MinRowId()));
+        row.SetField(8, OptionalInt64Value(manifest.MaxRowId()));
+        rows.push_back(std::move(row));
+    }
+    return rows;
+}
+
+FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path,
+                                   std::string branch, std::shared_ptr table_schema,
+                                   std::map options)
+    : InMemorySystemTable(table_path),
+      context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch),
+                                     std::move(table_schema), std::move(options))) {}
+
+std::string FilesSystemTable::Name() const {
+    return kName;
+}
+
+Result> FilesSystemTable::ArrowSchema() const {
+    return arrow::schema({
+        arrow::field("partition", arrow::utf8(), /*nullable=*/true),
+        arrow::field("bucket", arrow::int32(), /*nullable=*/false),
+        arrow::field("file_path", arrow::utf8(), /*nullable=*/false),
+        arrow::field("file_format", arrow::utf8(), /*nullable=*/false),
+        arrow::field("schema_id", arrow::int64(), /*nullable=*/false),
+        arrow::field("level", arrow::int32(), /*nullable=*/false),
+        arrow::field("record_count", arrow::int64(), /*nullable=*/false),
+        arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/false),
+        arrow::field("min_key", arrow::utf8(), /*nullable=*/true),
+        arrow::field("max_key", arrow::utf8(), /*nullable=*/true),
+        arrow::field("null_value_counts", arrow::utf8(), /*nullable=*/false),
+        arrow::field("min_value_stats", arrow::utf8(), /*nullable=*/false),
+        arrow::field("max_value_stats", arrow::utf8(), /*nullable=*/false),
+        arrow::field("min_sequence_number", arrow::int64(), /*nullable=*/true),
+        arrow::field("max_sequence_number", arrow::int64(), /*nullable=*/true),
+        arrow::field("creation_time", arrow::timestamp(arrow::TimeUnit::MILLI),
+                     /*nullable=*/true),
+        arrow::field("deleteRowCount", arrow::int64(), /*nullable=*/true),
+        arrow::field("file_source", arrow::utf8(), /*nullable=*/true),
+        arrow::field("first_row_id", arrow::int64(), /*nullable=*/true),
+        arrow::field("write_cols", arrow::list(arrow::utf8()), /*nullable=*/true),
+    });
+}
+
+Result> FilesSystemTable::BuildRows() const {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema());
+    std::shared_ptr pool = GetDefaultPool();
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory,
+                           CreatePathFactory(context_, core_options, pool));
+    PAIMON_ASSIGN_OR_RAISE(std::vector entries,
+                           ReadLatestDataFiles(context_, path_factory, core_options, pool));
+    std::shared_ptr arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(context_.table_schema->Fields());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, context_.table_schema->PartitionKeys()));
+    const std::vector& value_stats_fields = context_.table_schema->Fields();
+
+    std::vector rows;
+    rows.reserve(entries.size());
+    for (const auto& entry : entries) {
+        if (!(entry.Kind() == FileKind::Add())) {
+            continue;
+        }
+
+        const std::shared_ptr& file = entry.File();
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_schema,
+                               LoadDataSchema(context_, file->schema_id));
+        PAIMON_ASSIGN_OR_RAISE(std::vector data_stats_fields,
+                               ProjectWriteFields(data_schema, *file));
+        PAIMON_ASSIGN_OR_RAISE(std::vector key_fields,
+                               KeyFieldsForFilesTable(data_schema));
+        auto stats_evolution = std::make_shared(
+            data_stats_fields, value_stats_fields,
+            data_schema->Id() != context_.table_schema->Id() || file->write_cols.has_value(), pool);
+        PAIMON_ASSIGN_OR_RAISE(
+            SimpleStatsEvolution::EvolutionStats stats,
+            stats_evolution->Evolution(file->value_stats, file->row_count, file->value_stats_cols));
+        PAIMON_ASSIGN_OR_RAISE(StatsStringOverrides stats_overrides,
+                               OmittedPartitionStats(context_.table_schema, *file,
+                                                     entry.Partition(), file->row_count));
+
+        GenericRow row(schema->num_fields());
+        if (context_.table_schema->PartitionKeys().empty()) {
+            row.SetField(0, NullType());
+        } else {
+            PAIMON_ASSIGN_OR_RAISE(VariantType partition, OptionalPartitionStringValue(
+                                                              entry.Partition(), partition_schema));
+            row.SetField(0, partition);
+        }
+        row.SetField(1, entry.Bucket());
+        PAIMON_ASSIGN_OR_RAISE(std::string file_path, FilePath(path_factory, entry, *file));
+        row.SetField(2, StringValue(file_path));
+        PAIMON_ASSIGN_OR_RAISE(std::string file_format, file->FileFormat());
+        row.SetField(3, StringValue(file_format));
+        row.SetField(4, file->schema_id);
+        row.SetField(5, file->level);
+        row.SetField(6, file->row_count);
+        row.SetField(7, file->file_size);
+        PAIMON_ASSIGN_OR_RAISE(std::optional min_key,
+                               OptionalRowValuesString(key_fields, file->min_key, "[", "]"));
+        PAIMON_ASSIGN_OR_RAISE(std::optional max_key,
+                               OptionalRowValuesString(key_fields, file->max_key, "[", "]"));
+        row.SetField(8, OptionalStringValue(min_key));
+        row.SetField(9, OptionalStringValue(max_key));
+        PAIMON_ASSIGN_OR_RAISE(std::string null_value_counts,
+                               NullValueCountsString(value_stats_fields, *stats.null_counts,
+                                                     stats_overrides.null_counts));
+        row.SetField(10, StringValue(null_value_counts));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::string min_value_stats,
+            FieldsValueMapString(value_stats_fields, *stats.min_values, stats_overrides.values));
+        row.SetField(11, StringValue(min_value_stats));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::string max_value_stats,
+            FieldsValueMapString(value_stats_fields, *stats.max_values, stats_overrides.values));
+        row.SetField(12, StringValue(max_value_stats));
+        row.SetField(13, file->min_sequence_number);
+        row.SetField(14, file->max_sequence_number);
+        row.SetField(15, TimestampMillisValue(file->creation_time.GetMillisecond()));
+        row.SetField(16, OptionalInt64Value(file->delete_row_count));
+        row.SetField(17, file->file_source ? StringValue(file->file_source.value().ToString())
+                                           : VariantType(NullType()));
+        row.SetField(18, OptionalInt64Value(file->first_row_id));
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr write_cols,
+                               WriteColsValue(file->write_cols, pool));
+        row.SetField(19, write_cols ? VariantType(write_cols) : VariantType(NullType()));
+        rows.push_back(std::move(row));
+    }
+    return rows;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h
new file mode 100644
index 00000000..e3961e47
--- /dev/null
+++ b/src/paimon/core/table/system/metadata_system_tables.h
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/core/table/system/in_memory_system_table.h"
+
+namespace paimon {
+class FileSystem;
+class TableSchema;
+
+/// System table for `T$options`, exposing the latest base table options as key/value rows.
+class OptionsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "options";
+
+    OptionsSystemTable(std::string table_path, std::shared_ptr table_schema);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    std::shared_ptr table_schema_;
+};
+
+/// Shared table metadata location used by metadata system tables.
+struct MetadataSystemTableContext {
+    std::shared_ptr fs;
+    std::string table_path;
+    std::string branch;
+    std::shared_ptr table_schema;
+    std::map options;
+};
+
+/// System table for `T$snapshots`, exposing snapshot commit history.
+class SnapshotsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "snapshots";
+
+    SnapshotsSystemTable(std::shared_ptr fs, std::string table_path,
+                         std::string branch);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$schemas`, exposing schema evolution history.
+class SchemasSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "schemas";
+
+    SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$tags`, exposing tags and the snapshots they reference.
+class TagsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "tags";
+
+    TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$branches`, exposing table branches including `main`.
+class BranchesSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "branches";
+
+    BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$consumers`, exposing persisted streaming consumer offsets.
+class ConsumersSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "consumers";
+
+    ConsumersSystemTable(std::shared_ptr fs, std::string table_path,
+                         std::string branch);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$manifests`, exposing data manifest metadata in the latest snapshot.
+class ManifestsSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "manifests";
+
+    ManifestsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch,
+                         std::shared_ptr table_schema,
+                         std::map options);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+/// System table for `T$files`, exposing data file metadata in the latest snapshot.
+class FilesSystemTable : public InMemorySystemTable {
+ public:
+    static constexpr const char* kName = "files";
+
+    FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch,
+                     std::shared_ptr table_schema,
+                     std::map options);
+
+    std::string Name() const override;
+    Result> ArrowSchema() const override;
+    Result> BuildRows() const override;
+
+ private:
+    MetadataSystemTableContext context_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/options_system_table.cpp b/src/paimon/core/table/system/options_system_table.cpp
deleted file mode 100644
index 3d9c6859..00000000
--- a/src/paimon/core/table/system/options_system_table.cpp
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/table/system/options_system_table.h"
-
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include "arrow/api.h"
-#include "arrow/c/bridge.h"
-#include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/common/utils/arrow/mem_utils.h"
-#include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/core/schema/table_schema.h"
-#include "paimon/core/table/system/system_table_scan.h"
-#include "paimon/memory/memory_pool.h"
-#include "paimon/read_context.h"
-#include "paimon/result.h"
-#include "paimon/status.h"
-#include "paimon/table/source/table_read.h"
-
-namespace paimon {
-namespace {
-
-std::shared_ptr OptionsSchema() {
-    return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false),
-                          arrow::field("value", arrow::utf8(), /*nullable=*/false)});
-}
-
-class OptionsBatchReader : public BatchReader {
- public:
-    OptionsBatchReader(std::map options,
-                       const std::shared_ptr& pool)
-        : arrow_pool_(GetArrowPool(pool)), options_(std::move(options)) {}
-
-    Result NextBatch() override {
-        if (emitted_) {
-            return BatchReader::MakeEofBatch();
-        }
-        emitted_ = true;
-
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_array_builder,
-                                          arrow::MakeBuilder(arrow::utf8(), arrow_pool_.get()));
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr value_array_builder,
-                                          arrow::MakeBuilder(arrow::utf8(), arrow_pool_.get()));
-        auto* key_builder = dynamic_cast(key_array_builder.get());
-        auto* value_builder = dynamic_cast(value_array_builder.get());
-        if (key_builder == nullptr || value_builder == nullptr) {
-            return Status::Invalid("cannot create string builders for options system table");
-        }
-        for (const auto& [key, value] : options_) {
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(key));
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(value));
-        }
-        std::shared_ptr key_array;
-        std::shared_ptr value_array;
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Finish(&key_array));
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Finish(&value_array));
-        auto struct_array = std::make_shared(
-            arrow::struct_(OptionsSchema()->fields()), key_array->length(),
-            std::vector>{key_array, value_array});
-
-        auto c_array = std::make_unique<::ArrowArray>();
-        auto c_schema = std::make_unique<::ArrowSchema>();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportArray(*struct_array, c_array.get(), c_schema.get()));
-        return std::make_pair(std::move(c_array), std::move(c_schema));
-    }
-
-    std::shared_ptr GetReaderMetrics() const override {
-        return std::make_shared();
-    }
-
-    void Close() override {
-        emitted_ = true;
-    }
-
- private:
-    std::unique_ptr arrow_pool_;
-    std::map options_;
-    bool emitted_ = false;
-};
-
-class OptionsTableRead : public TableRead {
- public:
-    OptionsTableRead(std::map options,
-                     const std::shared_ptr& memory_pool)
-        : TableRead(memory_pool), options_(std::move(options)) {}
-
-    Result> CreateReader(
-        const std::vector>& splits) override {
-        if (splits.size() != 1) {
-            return Status::Invalid("options system table expects a single split");
-        }
-        for (const auto& split : splits) {
-            if (!std::dynamic_pointer_cast(split)) {
-                return Status::Invalid("unsupported split for options system table");
-            }
-        }
-        return std::make_unique(options_, GetMemoryPool());
-    }
-
-    Result> CreateReader(
-        const std::shared_ptr& split) override {
-        std::vector> splits = {split};
-        return CreateReader(splits);
-    }
-
- private:
-    std::map options_;
-};
-
-}  // namespace
-
-OptionsSystemTable::OptionsSystemTable(std::string table_path,
-                                       std::shared_ptr table_schema)
-    : table_path_(std::move(table_path)), table_schema_(std::move(table_schema)) {}
-
-std::string OptionsSystemTable::Name() const {
-    return kName;
-}
-
-Result> OptionsSystemTable::ArrowSchema() const {
-    return OptionsSchema();
-}
-
-Result> OptionsSystemTable::NewScan(
-    const std::shared_ptr& /*context*/) const {
-    return std::make_unique(table_path_);
-}
-
-Result> OptionsSystemTable::NewRead(
-    const std::shared_ptr& context) const {
-    return std::make_unique(table_schema_->Options(), context->GetMemoryPool());
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp
new file mode 100644
index 00000000..516861cb
--- /dev/null
+++ b/src/paimon/core/table/system/read_optimized_system_table.cpp
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/system/read_optimized_system_table.h"
+
+#include 
+#include 
+#include 
+
+#include "arrow/c/bridge.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/defs.h"
+#include "paimon/read_context.h"
+#include "paimon/scan_context.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+
+namespace paimon {
+
+ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path,
+                                                   std::shared_ptr table_schema,
+                                                   std::map options)
+    : table_path_(std::move(table_path)),
+      table_schema_(std::move(table_schema)),
+      options_(std::move(options)) {}
+
+std::string ReadOptimizedSystemTable::Name() const {
+    return kName;
+}
+
+Result> ReadOptimizedSystemTable::ArrowSchema() const {
+    return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields());
+}
+
+std::map ReadOptimizedSystemTable::ReadOptimizedOptions() const {
+    auto options = options_;
+    options[kReadOptimizedScanOption] = "true";
+    return options;
+}
+
+Result> ReadOptimizedSystemTable::NewScan(
+    const std::shared_ptr& context) const {
+    auto options = ReadOptimizedOptions();
+    ScanContextBuilder builder(table_path_);
+    builder.SetOptions(options)
+        .WithStreamingMode(context->IsStreamingMode())
+        .WithMemoryPool(context->GetMemoryPool())
+        .WithExecutor(context->GetExecutor())
+        .WithFileSystem(context->GetSpecificFileSystem())
+        .WithCache(context->GetCache());
+    if (context->GetLimit().has_value()) {
+        builder.SetLimit(context->GetLimit().value());
+    }
+    if (context->GetScanFilters()) {
+        if (context->GetScanFilters()->GetBucketFilter().has_value()) {
+            builder.SetBucketFilter(context->GetScanFilters()->GetBucketFilter().value());
+        }
+        builder.SetPartitionFilter(context->GetScanFilters()->GetPartitionFilters());
+        builder.SetPredicate(context->GetScanFilters()->GetPredicate());
+    }
+    if (context->GetGlobalIndexResult()) {
+        builder.SetGlobalIndexResult(context->GetGlobalIndexResult());
+    }
+    if (context->GetSpecificTableSchema().has_value()) {
+        builder.SetTableSchema(context->GetSpecificTableSchema().value());
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish());
+    return TableScan::Create(std::move(base_context));
+}
+
+Result> ReadOptimizedSystemTable::NewRead(
+    const std::shared_ptr& context) const {
+    auto options = options_;
+    std::string branch = context->GetBranch();
+    // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`.
+    auto branch_iter = options.find(Options::BRANCH);
+    if (branch_iter != options.end()) {
+        branch = branch_iter->second;
+    }
+    ReadContextBuilder builder(table_path_);
+    builder.SetOptions(options)
+        .WithBranch(branch)
+        .SetPredicate(context->GetPredicate())
+        .EnablePredicateFilter(context->EnablePredicateFilter())
+        .EnablePrefetch(context->EnablePrefetch())
+        .SetPrefetchBatchCount(context->GetPrefetchBatchCount())
+        .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum())
+        .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch())
+        .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber())
+        .WithMemoryPool(context->GetMemoryPool())
+        .WithExecutor(context->GetExecutor())
+        .WithFileSystem(context->GetSpecificFileSystem())
+        .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap())
+        .SetPrefetchCacheMode(context->GetPrefetchCacheMode())
+        .WithCacheConfig(context->GetCacheConfig())
+        .WithCache(context->GetCache())
+        .SetReadFieldNames(context->GetReadFieldNames())
+        .SetReadFieldIds(context->GetReadFieldIds());
+    if (context->HasReadSchema()) {
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema,
+                                          arrow::ImportSchema(context->GetReadSchema()));
+        auto c_read_schema = std::make_unique<::ArrowSchema>();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get()));
+        builder.SetReadSchema(std::move(c_read_schema));
+    }
+    if (context->GetSpecificTableSchema().has_value()) {
+        builder.SetTableSchema(context->GetSpecificTableSchema().value());
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish());
+    return TableRead::Create(std::move(base_context));
+}
+
+}  // namespace paimon
diff --git a/src/paimon/core/table/system/options_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h
similarity index 75%
rename from src/paimon/core/table/system/options_system_table.h
rename to src/paimon/core/table/system/read_optimized_system_table.h
index 2282721d..14fadbfd 100644
--- a/src/paimon/core/table/system/options_system_table.h
+++ b/src/paimon/core/table/system/read_optimized_system_table.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include 
 #include 
 #include 
 
@@ -27,11 +28,13 @@
 namespace paimon {
 class TableSchema;
 
-class OptionsSystemTable : public SystemTable {
+/// System table for `T$ro`, exposing read-optimized data.
+class ReadOptimizedSystemTable : public SystemTable {
  public:
-    static constexpr const char* kName = "options";
+    static constexpr const char* kName = "ro";
 
-    OptionsSystemTable(std::string table_path, std::shared_ptr table_schema);
+    ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema,
+                             std::map options);
 
     std::string Name() const override;
     Result> ArrowSchema() const override;
@@ -41,8 +44,11 @@ class OptionsSystemTable : public SystemTable {
         const std::shared_ptr& context) const override;
 
  private:
+    std::map ReadOptimizedOptions() const;
+
     std::string table_path_;
     std::shared_ptr table_schema_;
+    std::map options_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp
index 676d2a0f..2d67355a 100644
--- a/src/paimon/core/table/system/system_table.cpp
+++ b/src/paimon/core/table/system/system_table.cpp
@@ -19,11 +19,14 @@
 
 #include "paimon/core/table/system/system_table.h"
 
+#include 
 #include 
 #include 
 #include 
 #include 
+#include 
 
+#include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/string_utils.h"
@@ -31,42 +34,176 @@
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/system/audit_log_system_table.h"
 #include "paimon/core/table/system/binlog_system_table.h"
-#include "paimon/core/table/system/options_system_table.h"
+#include "paimon/core/table/system/global_system_tables.h"
+#include "paimon/core/table/system/metadata_system_tables.h"
+#include "paimon/core/table/system/read_optimized_system_table.h"
 #include "paimon/core/utils/branch_manager.h"
 #include "paimon/status.h"
 
 namespace paimon {
+namespace {
 
-bool SystemTableLoader::IsSupported(const std::string& system_table_name) {
-    std::string normalized_name = StringUtils::ToLowerCase(system_table_name);
-    return normalized_name == OptionsSystemTable::kName ||
-           normalized_name == AuditLogSystemTable::kName ||
-           normalized_name == BinlogSystemTable::kName;
-}
+using SystemTableFactory = std::function>(
+    const std::shared_ptr&, const std::string&, const std::shared_ptr&,
+    const std::map&)>;
 
-Result> SystemTableLoader::Load(
-    const std::string& system_table_name, const std::shared_ptr& fs,
-    const std::string& table_path, const std::shared_ptr& table_schema,
+struct SystemTableRegistryEntry {
+    std::string name;
+    SystemTableFactory factory;
+};
+
+std::map MergeOptions(
+    const std::shared_ptr& table_schema,
     const std::map& dynamic_options) {
-    std::string normalized_name = StringUtils::ToLowerCase(system_table_name);
-    if (normalized_name == OptionsSystemTable::kName) {
-        return std::make_shared(table_path, table_schema);
-    }
     auto options = table_schema->Options();
     for (const auto& [key, value] : dynamic_options) {
         options[key] = value;
     }
-    if (normalized_name == AuditLogSystemTable::kName) {
-        return std::make_shared(fs, table_path, table_schema, options);
+    return options;
+}
+
+std::string LoadBranch(const std::map& options) {
+    auto branch_iter = options.find(Options::BRANCH);
+    return branch_iter == options.end() ? BranchManager::DEFAULT_MAIN_BRANCH : branch_iter->second;
+}
+
+const std::vector& SystemTableRegistry() {
+    static const std::vector registry = {
+        {OptionsSystemTable::kName,
+         [](const std::shared_ptr& /*fs*/, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& /*dynamic_options*/)
+             -> Result> {
+             return std::make_shared(table_path, table_schema);
+         }},
+        {AuditLogSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             return std::make_shared(
+                 fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options));
+         }},
+        {BinlogSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             return std::make_shared(
+                 fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options));
+         }},
+        {ReadOptimizedSystemTable::kName,
+         [](const std::shared_ptr& /*fs*/, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             return std::make_shared(
+                 table_path, table_schema, MergeOptions(table_schema, dynamic_options));
+         }},
+        {SnapshotsSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             return std::make_shared(fs, table_path, LoadBranch(options));
+         }},
+        {SchemasSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             return std::make_shared(fs, table_path, LoadBranch(options));
+         }},
+        {TagsSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             return std::make_shared(fs, table_path, LoadBranch(options));
+         }},
+        {BranchesSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             return std::make_shared(fs, table_path, LoadBranch(options));
+         }},
+        {ConsumersSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             return std::make_shared(fs, table_path, LoadBranch(options));
+         }},
+        {ManifestsSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             auto branch = LoadBranch(options);
+             return std::make_shared(fs, table_path, std::move(branch),
+                                                           table_schema, std::move(options));
+         }},
+        {FilesSystemTable::kName,
+         [](const std::shared_ptr& fs, const std::string& table_path,
+            const std::shared_ptr& table_schema,
+            const std::map& dynamic_options)
+             -> Result> {
+             auto options = MergeOptions(table_schema, dynamic_options);
+             auto branch = LoadBranch(options);
+             return std::make_shared(fs, table_path, std::move(branch),
+                                                       table_schema, std::move(options));
+         }},
+    };
+    return registry;
+}
+
+std::optional FindSystemTableFactory(const std::string& system_table_name) {
+    std::string normalized_name = StringUtils::ToLowerCase(system_table_name);
+    for (const auto& entry : SystemTableRegistry()) {
+        if (entry.name == normalized_name) {
+            return entry.factory;
+        }
     }
-    if (normalized_name == BinlogSystemTable::kName) {
-        return std::make_shared(fs, table_path, table_schema, options);
+    return std::nullopt;
+}
+
+}  // namespace
+
+bool SystemTableLoader::IsSupported(const std::string& system_table_name) {
+    return FindSystemTableFactory(system_table_name).has_value();
+}
+
+Result> SystemTableLoader::Load(
+    const std::string& system_table_name, const std::shared_ptr& fs,
+    const std::string& table_path, const std::shared_ptr& table_schema,
+    const std::map& dynamic_options) {
+    std::optional factory = FindSystemTableFactory(system_table_name);
+    if (factory) {
+        return factory.value()(fs, table_path, table_schema, dynamic_options);
     }
     return Status::NotImplemented("unsupported system table: ", system_table_name);
 }
 
 Result> SystemTableLoader::TryParsePath(const std::string& path) {
     std::string table_name = PathUtil::GetName(path);
+    std::string parent = PathUtil::GetParentDirPath(path);
+    std::string parent_name = PathUtil::GetName(parent);
+
+    // Detect global system table paths: /sys/
+    if (parent_name == Catalog::SYSTEM_DATABASE_NAME) {
+        SystemTablePath system_table_path;
+        system_table_path.is_global = true;
+        system_table_path.system_table_name = table_name;
+        return std::optional(std::move(system_table_path));
+    }
+
     Identifier identifier(table_name);
     PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable());
     if (!is_system_table) {
@@ -76,7 +213,6 @@ Result> SystemTableLoader::TryParsePath(const std
     PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName());
     PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name,
                            identifier.GetSystemTableName());
-    std::string parent = PathUtil::GetParentDirPath(path);
     SystemTablePath system_table_path;
     system_table_path.table_path = PathUtil::JoinPath(parent, data_table_name);
     system_table_path.branch = std::move(branch);
@@ -92,6 +228,21 @@ Result> SystemTableLoader::LoadFromPath(
         return Status::Invalid("path is not a system table path: ", path);
     }
     const auto& parsed = system_table_path.value();
+
+    // Handle global system tables (under sys/ directory)
+    if (parsed.is_global) {
+        GlobalSystemTableContext context;
+        context.fs = fs;
+        // The warehouse is the grandparent of the sys/
path + context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); + context.catalog_options = dynamic_options; + // Note: context.catalog is intentionally left as nullptr here. + // Global tables loaded from path do not have a Catalog reference and + // cannot enumerate databases/tables. Only tables that don't require + // catalog enumeration (e.g. catalog_options) will work in this path. + return GlobalSystemTableLoader::Load(parsed.system_table_name, context); + } + SchemaManager schema_manager(fs, parsed.table_path, parsed.branch.value_or(BranchManager::DEFAULT_MAIN_BRANCH)); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 16e38780..39e1da31 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -37,12 +37,21 @@ class TableScan; class TableRead; class TableSchema; +/// Parsed information for a table-scoped system table path. struct SystemTablePath { + /// Base data table path without the system table suffix. std::string table_path; + /// Optional branch parsed from identifiers such as `T$branch_dev$options`. std::optional branch; + /// System table name, for example `options` or `snapshots`. std::string system_table_name; + /// Whether this is a global system table under the `sys` database. + bool is_global = false; }; +/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// +/// Implementations expose a read-only schema and create their own scan/read objects. class SystemTable : public std::enable_shared_from_this { public: virtual ~SystemTable() = default; @@ -55,6 +64,9 @@ class SystemTable : public std::enable_shared_from_this { const std::shared_ptr& context) const = 0; }; +/// Loads system table implementations from parsed table identifiers or table paths. +/// +/// The loader owns the registry that maps a system table name to its factory. class SystemTableLoader { public: static bool IsSupported(const std::string& system_table_name); diff --git a/src/paimon/core/table/system/system_table_scan.h b/src/paimon/core/table/system/system_table_scan.h index 33201d80..d42e7dd5 100644 --- a/src/paimon/core/table/system/system_table_scan.h +++ b/src/paimon/core/table/system/system_table_scan.h @@ -29,6 +29,7 @@ namespace paimon { class Plan; class Split; +/// Singleton split used by in-memory system tables. class SystemTableSplit : public Split { public: explicit SystemTableSplit(const std::string& table_path) : table_path_(table_path) {} @@ -41,6 +42,7 @@ class SystemTableSplit : public Split { std::string table_path_; }; +/// Scan implementation for system tables that materialize exactly one split. class SystemTableScan : public TableScan { public: explicit SystemTableScan(const std::string& table_path); diff --git a/src/paimon/core/table/system/system_table_schema.h b/src/paimon/core/table/system/system_table_schema.h index e735d0a8..14e3d19c 100644 --- a/src/paimon/core/table/system/system_table_schema.h +++ b/src/paimon/core/table/system/system_table_schema.h @@ -31,6 +31,7 @@ namespace paimon { +/// Read-only schema wrapper for system tables. class SystemTableSchema : public SystemSchema { public: explicit SystemTableSchema(std::shared_ptr schema); diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index eb4bb139..0c67ae3a 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -17,19 +17,31 @@ * under the License. */ +#include "paimon/core/table/system/system_table.h" + #include #include +#include #include +#include +#include #include "arrow/api.h" +#include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" +#include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/fs/file_system_factory.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" #include "paimon/result.h" #include "paimon/status.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -65,4 +77,126 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { "Invalid Config [table-read.sequence-number.enabled: invalid]"); } +TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) { + std::map options = { + {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + + BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema()); + + ASSERT_EQ(schema->field_names(), + (std::vector{"rowkind", "_SEQUENCE_NUMBER", "pk", "v"})); + ASSERT_EQ(schema->field(0)->type()->id(), arrow::Type::STRING); + ASSERT_FALSE(schema->field(0)->nullable()); + ASSERT_EQ(schema->field(1)->type()->id(), arrow::Type::INT64); + ASSERT_EQ(schema->field(2)->type()->id(), arrow::Type::LIST); + ASSERT_EQ(schema->field(3)->type()->id(), arrow::Type::LIST); +} + +TEST(SystemTableTest, TestStreamingBinlogPacksUpdateAcrossBatches) { + std::shared_ptr input_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }); + std::shared_ptr input = + arrow::ipc::internal::json::ArrayFromJSON( + input_type, R"([[0, 10, "a", 1], [1, 11, "b", 2], [2, 12, "b", 3], [3, 13, "d", 4]])") + .ValueOrDie(); + std::shared_ptr output_schema = arrow::schema({ + arrow::field("rowkind", arrow::utf8(), /*nullable=*/false), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + arrow::field("pk", arrow::list(arrow::utf8())), + arrow::field("v", arrow::list(arrow::int32())), + }); + std::unique_ptr reader = CreateChangelogBatchReader( + std::make_unique(input, input_type, /*read_batch_size=*/2), + output_schema, + /*include_sequence_number=*/true, CreateBinlogBatchConverter(), + /*pack_update_before_after=*/true, GetDefaultPool()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(reader.get())); + std::shared_ptr expected_array = + arrow::ipc::internal::json::ArrayFromJSON(actual->type(), R"([ + ["+I", 10, ["a"], [1]], + ["+U", 11, ["b", "b"], [2, 3]], + ["-D", 13, ["d"], [4]] + ])") + .ValueOrDie(); + auto expected = std::make_shared(expected_array); + ASSERT_TRUE(actual->Equals(*expected)) + << "expected: " << expected->ToString() << "\nactual: " << actual->ToString(); +} + +TEST(SystemTableTest, TestStreamingBinlogEmitsUnmatchedUpdateBefore) { + std::shared_ptr input_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }); + std::shared_ptr input = + arrow::ipc::internal::json::ArrayFromJSON(input_type, R"([[1, "b", 2]])").ValueOrDie(); + std::shared_ptr output_schema = arrow::schema({ + arrow::field("rowkind", arrow::utf8(), /*nullable=*/false), + arrow::field("pk", arrow::list(arrow::utf8())), + arrow::field("v", arrow::list(arrow::int32())), + }); + std::unique_ptr reader = CreateChangelogBatchReader( + std::make_unique(input, input_type, /*read_batch_size=*/1), + output_schema, + /*include_sequence_number=*/false, CreateBinlogBatchConverter(), + /*pack_update_before_after=*/true, GetDefaultPool()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(reader.get())); + std::shared_ptr expected_array = + arrow::ipc::internal::json::ArrayFromJSON(actual->type(), R"([ + ["-U", ["b"], [2]] + ])") + .ValueOrDie(); + auto expected = std::make_shared(expected_array); + ASSERT_TRUE(actual->Equals(*expected)) + << "expected: " << expected->ToString() << "\nactual: " << actual->ToString(); +} + +TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { + ASSERT_TRUE(SystemTableLoader::IsSupported(ReadOptimizedSystemTable::kName)); + + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); + ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); + ASSERT_EQ(arrow_schema->field(0)->type()->id(), arrow::Type::STRING); + ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); +} + +TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { + ASSERT_OK_AND_ASSIGN(std::optional parsed, + SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); + ASSERT_TRUE(parsed.has_value()); + ASSERT_EQ(parsed->table_path, "/tmp/db.db/t"); + ASSERT_TRUE(parsed->branch.has_value()); + ASSERT_EQ(parsed->branch.value(), "audit"); + ASSERT_EQ(parsed->system_table_name, ReadOptimizedSystemTable::kName); +} + +TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) { + ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", "/tmp", {})); + std::shared_ptr shared_fs(std::move(fs)); + ASSERT_NOK_WITH_MSG(SystemTableLoader::LoadFromPath(shared_fs, "/tmp/warehouse/sys/tables", {}), + "global system table requires catalog context: tables"); +} + } // namespace paimon::test diff --git a/src/paimon/core/tag/tag.cpp b/src/paimon/core/tag/tag.cpp index ebf77737..6ee31864 100644 --- a/src/paimon/core/tag/tag.cpp +++ b/src/paimon/core/tag/tag.cpp @@ -39,9 +39,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, const int64_t commit_identifier, const CommitKind commit_kind, const int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + const int64_t total_record_count, const int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -51,7 +49,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t : Snapshot(version, id, schema_id, base_manifest_list, base_manifest_list_size, delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, - commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + commit_kind, time_millis, total_record_count, delta_record_count, changelog_record_count, watermark, statistics, properties, next_row_id), tag_create_time_(tag_create_time), tag_time_retained_(tag_time_retained) {} @@ -77,9 +75,8 @@ Result Tag::TrimToSnapshot() const { return Snapshot(Version(), Id(), SchemaId(), BaseManifestList(), BaseManifestListSize(), DeltaManifestList(), DeltaManifestListSize(), ChangelogManifestList(), ChangelogManifestListSize(), IndexManifest(), CommitUser(), CommitIdentifier(), - GetCommitKind(), TimeMillis(), LogOffsets(), TotalRecordCount(), - DeltaRecordCount(), ChangelogRecordCount(), Watermark(), Statistics(), - Properties(), NextRowId()); + GetCommitKind(), TimeMillis(), TotalRecordCount(), DeltaRecordCount(), + ChangelogRecordCount(), Watermark(), Statistics(), Properties(), NextRowId()); } rapidjson::Value Tag::ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) { diff --git a/src/paimon/core/tag/tag.h b/src/paimon/core/tag/tag.h index 23508e58..5e84abb3 100644 --- a/src/paimon/core/tag/tag.h +++ b/src/paimon/core/tag/tag.h @@ -52,9 +52,7 @@ class Tag : public Snapshot { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, diff --git a/src/paimon/core/tag/tag_test.cpp b/src/paimon/core/tag/tag_test.cpp index fb371e82..c94d823b 100644 --- a/src/paimon/core/tag/tag_test.cpp +++ b/src/paimon/core/tag/tag_test.cpp @@ -34,6 +34,15 @@ class TagTest : public testing::Test { std::string replaced_str = StringUtils::Replace(str, " ", ""); replaced_str = StringUtils::Replace(replaced_str, "\t", ""); replaced_str = StringUtils::Replace(replaced_str, "\n", ""); + // logOffsets was removed from snapshot json; normalize legacy fixtures. + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{},", ""); + replaced_str = StringUtils::Replace(replaced_str, ",\"logOffsets\":{}", ""); + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{}", ""); + // changelogManifestList is @JsonInclude(NON_NULL) in Java and omitted on serialization; + // strip the stale null field from checked-in Java fixtures before comparing. + replaced_str = StringUtils::Replace(replaced_str, "\"changelogManifestList\":null,", ""); + replaced_str = StringUtils::Replace(replaced_str, ",\"changelogManifestList\":null", ""); + replaced_str = StringUtils::Replace(replaced_str, "\"changelogManifestList\":null", ""); if (serialized) { replaced_str = StringUtils::Replace(replaced_str, ".0", ".000000000"); } @@ -42,7 +51,6 @@ class TagTest : public testing::Test { }; TEST_F(TagTest, TestSimple) { - const std::map log_offset = {{25, 30}}; const std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; const auto tag_create_time = std::vector({2026, 1, 2, 3, 4, 5, 6}); const Tag tag( @@ -51,7 +59,7 @@ TEST_F(TagTest, TestSimple) { /*changelog_manifest_list=*/"changelog_manifest_list", 30, /*index_manifest=*/"index_manifest", /*commit_user=*/"commit_user_01", /*commit_identifier=*/20, - /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset, + /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50, /*statistics=*/"statistic_test", properties, /*next_row_id=*/0, @@ -70,9 +78,8 @@ TEST_F(TagTest, TestSimple) { ASSERT_EQ(20, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Compact(), tag.GetCommitKind()); ASSERT_EQ(1234, tag.TimeMillis()); - ASSERT_EQ(log_offset, tag.LogOffsets().value()); - ASSERT_EQ(35, tag.TotalRecordCount().value()); - ASSERT_EQ(40, tag.DeltaRecordCount().value()); + ASSERT_EQ(35, tag.TotalRecordCount()); + ASSERT_EQ(40, tag.DeltaRecordCount()); ASSERT_EQ(45, tag.ChangelogRecordCount().value()); ASSERT_EQ(50, tag.Watermark().value()); ASSERT_EQ("statistic_test", tag.Statistics().value()); @@ -101,9 +108,8 @@ TEST_F(TagTest, TestFromPath) { ASSERT_EQ(9223372036854775807ll, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Append(), tag.GetCommitKind()); ASSERT_EQ(1721614343270ll, tag.TimeMillis()); - ASSERT_EQ((std::map()), tag.LogOffsets().value()); - ASSERT_EQ(5, tag.TotalRecordCount().value()); - ASSERT_EQ(5, tag.DeltaRecordCount().value()); + ASSERT_EQ(5, tag.TotalRecordCount()); + ASSERT_EQ(5, tag.DeltaRecordCount()); ASSERT_EQ(0, tag.ChangelogRecordCount().value()); ASSERT_EQ(std::nullopt, tag.Watermark()); ASSERT_EQ(std::nullopt, tag.Statistics()); @@ -122,12 +128,10 @@ TEST_F(TagTest, TestJsonizable) { "baseManifestListSize" : 20, "deltaManifestList" : "manifest-list-d96fcc30-99e8-4f45-962b-a1157c56f378-1", "deltaManifestListSize" : 50, - "changelogManifestList" : null, "commitUser" : "0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", "timeMillis" : 1711692199281, - "logOffsets" : { }, "totalRecordCount" : 3, "deltaRecordCount" : 3, "changelogRecordCount" : 0, @@ -147,7 +151,6 @@ TEST_F(TagTest, TestJsonizable) { /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", /*commit_identifier=*/9223372036854775807ll, /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll, - /*log_offsets=*/std::map(), /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt, @@ -190,15 +193,10 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -221,15 +219,10 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp new file mode 100644 index 00000000..cf41ca81 --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/blob_view_lookup.h" + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/executor/future.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/defs.h" +#include "paimon/executor.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/memory/bytes.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/utils/special_field_ids.h" + +namespace paimon { +BlobViewLookup::TableReadPlan::TableReadPlan(const BlobViewStruct& view_struct) + : identifier_(view_struct.GetIdentifier()) { + references_by_field_id_.insert(view_struct.FieldId()); + row_ranges_.push_back(view_struct.RowId()); +} + +void BlobViewLookup::TableReadPlan::Add(const BlobViewStruct& view_struct) { + references_by_field_id_.insert(view_struct.FieldId()); + row_ranges_.push_back(view_struct.RowId()); +} + +const Identifier& BlobViewLookup::TableReadPlan::GetIdentifier() const { + return identifier_; +} + +std::vector BlobViewLookup::TableReadPlan::GetFieldIds() const { + return std::vector(references_by_field_id_.begin(), references_by_field_id_.end()); +} + +std::vector BlobViewLookup::TableReadPlan::GetSortedDistinctRanges() const { + if (row_ranges_.empty()) { + return {}; + } + std::vector sorted = row_ranges_; + std::sort(sorted.begin(), sorted.end()); + std::vector ranges; + int64_t range_start = sorted[0]; + int64_t range_end = range_start; + for (size_t i = 1; i < sorted.size(); ++i) { + const int64_t row_id = sorted[i]; + if (row_id == range_end) { + continue; + } + if (row_id != range_end + 1) { + ranges.emplace_back(range_start, range_end); + range_start = row_id; + } + range_end = row_id; + } + ranges.emplace_back(range_start, range_end); + return ranges; +} + +Result BlobViewLookup::CreateResolver( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, const std::shared_ptr& pool, + const std::shared_ptr& executor) { + PAIMON_ASSIGN_OR_RAISE(DescriptorMapping mapping, + PreloadDescriptors(view_structs, catalog_context, pool, executor)); + return BlobViewResolver([cached = std::move(mapping)](const BlobViewStruct& view_struct) + -> Result> { + auto iter = cached.find(view_struct); + if (iter == cached.end()) { + return Status::Invalid(fmt::format("BlobViewStruct not found in preloaded cache: {}", + view_struct.ToString())); + } + return iter->second; + }); +} + +Result BlobViewLookup::PreloadDescriptors( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, const std::shared_ptr& pool, + const std::shared_ptr& executor) { + std::unordered_map plan_by_identifier = + GroupByIdentifier(view_structs); + int64_t target_rows_per_task = TargetRowsPerTask(plan_by_identifier, executor->GetThreadNum()); + + std::vector>> futures; + for (const auto& [identifier, table_read_plan] : plan_by_identifier) { + const auto& id = identifier; + std::vector field_ids = table_read_plan.GetFieldIds(); + std::vector> range_chunks = + SplitRowRanges(table_read_plan.GetSortedDistinctRanges(), target_rows_per_task); + for (const auto& range_chunk : range_chunks) { + futures.push_back(Via( + executor.get(), + [catalog_context, id, field_ids, range_chunk, pool]() -> Result { + return LoadTableDescriptorChunk(catalog_context, id, field_ids, range_chunk, + pool); + })); + } + } + + DescriptorMapping mapping; + std::vector> chunk_results = CollectAll(futures); + for (auto& chunk_result : chunk_results) { + if (!chunk_result.ok()) { + return chunk_result.status(); + } + for (const auto& [view_struct, descriptor] : chunk_result.value()) { + mapping[view_struct] = descriptor; + } + } + return mapping; +} + +Result BlobViewLookup::LoadTableDescriptorChunk( + const std::shared_ptr& catalog_context, const Identifier& identifier, + const std::vector& field_ids, const std::vector& row_ranges, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid("do not support upstream table with branch"); + } + auto file_system = catalog_context->file_system; + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(catalog_context, identifier)); + ScanContextBuilder scan_builder(table_path); + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + scan_builder.SetGlobalIndexResult(global_index_result) + .WithMemoryPool(pool) + .WithFileSystem(file_system); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_builder(table_path); + std::vector read_field_ids = field_ids; + read_field_ids.push_back(SpecialFieldIds::ROW_ID); + read_builder.SetReadFieldIds(read_field_ids) + .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") + .EnablePrefetch(true) + .WithMemoryPool(pool) + .WithFileSystem(file_system); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + table_read->CreateReader(plan->Splits())); + + DescriptorMapping mapping; + PAIMON_RETURN_NOT_OK( + ExtractBlobDescriptors(identifier, read_field_ids, pool, reader.get(), &mapping)); + return mapping; +} + +Result BlobViewLookup::GetTableLocation( + const std::shared_ptr& catalog_context, const Identifier& identifier) { + auto file_system = catalog_context->file_system; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr catalog, + Catalog::Create(catalog_context->root_path, catalog_context->options, file_system)); + // The table path may be either xxx/test_database/test_table or + // xxx/test_database.db/test_table. If neither path exists or both paths exist, it means we + // cannot infer the table path, and an error will be reported. If only one of the paths + // exists, we will use that path. + PAIMON_ASSIGN_OR_RAISE(std::string source_table_path, catalog->GetTableLocation(identifier)); + std::string database_name = identifier.GetDatabaseName(); + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + std::string fallback_source_table_path = PathUtil::JoinPath( + PathUtil::JoinPath(catalog_context->root_path, database_name), data_table_name); + + PAIMON_ASSIGN_OR_RAISE(bool exist, catalog_context->file_system->Exists(source_table_path)); + PAIMON_ASSIGN_OR_RAISE(bool fallback_exist, file_system->Exists(fallback_source_table_path)); + if (exist == fallback_exist) { + return Status::Invalid( + fmt::format("Ambiguous table path: both table path {} and fallback table path {} are " + "present or absent", + source_table_path, fallback_source_table_path)); + } + std::string final_table_path = exist ? source_table_path : fallback_source_table_path; + return final_table_path; +} + +Status BlobViewLookup::ExtractBlobDescriptors(const Identifier& identifier, + const std::vector& field_ids, + const std::shared_ptr& pool, + BatchReader* reader, DescriptorMapping* mapping) { + if (reader == nullptr) { + return Status::Invalid("invalid reader in ExtractBlobDescriptors, reader is nullptr"); + } + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (struct_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, batch array is not a StructArray."); + } + // skip the _VALUE_KIND column + if (static_cast(struct_array->num_fields()) - 1 != field_ids.size()) { + return Status::Invalid( + fmt::format("invalid array in ExtractBlobDescriptors, batch array fields(exclude " + "_VALUE_KIND) {} mismatch read field ids {}.", + struct_array->num_fields() - 1, field_ids.size())); + } + // get _VALUE_KIND + if (struct_array->struct_type()->field(0)->name() != SpecialFields::ValueKind().Name()) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, expected _VALUE_KIND as the first " + "column"); + } + + // get _ROW_ID + if (struct_array->struct_type()->field(struct_array->num_fields() - 1)->name() != + SpecialFields::RowId().Name()) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, expected _ROW_ID as the last column"); + } + auto row_id_array = struct_array->field(struct_array->num_fields() - 1); + auto typed_row_id_array = std::dynamic_pointer_cast(row_id_array); + if (!typed_row_id_array) { + return Status::Invalid( + fmt::format("invalid array does not contain {} field, or it cannot be casted to " + "Int64Array in ExtractBlobDescriptors.", + SpecialFields::RowId().Name())); + } + + // skip _VALUE_KIND + for (int32_t idx = 1; idx < struct_array->num_fields() - 1; ++idx) { + auto binary_array = + std::dynamic_pointer_cast(struct_array->field(idx)); + if (binary_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, column is not a LargeBinaryArray."); + } + for (int64_t row = 0; row < binary_array->length(); ++row) { + BlobViewStruct blob_view_struct(identifier, field_ids[idx - 1], + typed_row_id_array->Value(row)); + if (binary_array->IsNull(row)) { + // null in source table + (*mapping)[blob_view_struct] = nullptr; + continue; + } + std::string_view bytes = binary_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, BlobDescriptor::IsBlobDescriptor( + bytes.data(), bytes.size())); + if (!is_descriptor) { + return Status::Invalid( + "requires blob field value to be a serialized BlobDescriptor in source " + "table."); + } + auto descriptor_bytes = std::make_shared(bytes.size(), pool.get()); + std::memcpy(descriptor_bytes->data(), bytes.data(), bytes.size()); + (*mapping)[blob_view_struct] = std::move(descriptor_bytes); + } + } + } + return Status::OK(); +} + +std::unordered_map BlobViewLookup::GroupByIdentifier( + const std::unordered_set& view_structs) { + std::unordered_map grouped; + for (const auto& view_struct : view_structs) { + auto identifier = view_struct.GetIdentifier(); + auto iter = grouped.find(identifier); + if (iter != grouped.end()) { + iter->second.Add(view_struct); + } else { + grouped.emplace(identifier, BlobViewLookup::TableReadPlan(view_struct)); + } + } + return grouped; +} + +int64_t BlobViewLookup::TargetRowsPerTask( + const std::unordered_map& plan_by_identifier, uint32_t thread_num) { + int64_t total_rows = 0; + for (const auto& [identifier, table_read_plan] : plan_by_identifier) { + for (const auto& row_range : table_read_plan.GetSortedDistinctRanges()) { + total_rows += row_range.Count(); + } + } + int64_t balanced_rows = (total_rows + thread_num - 1) / thread_num; + return std::max(MIN_ROW_PER_TASK, balanced_rows); +} + +std::vector> BlobViewLookup::SplitRowRanges(const std::vector& row_ranges, + int64_t target_rows_per_task) { + if (row_ranges.empty()) { + return {}; + } + std::vector> chunks; + std::vector current_chunk; + int64_t current_chunk_rows = 0; + for (const auto& row_range : row_ranges) { + int64_t next_from = row_range.from; + while (next_from <= row_range.to) { + if (current_chunk_rows == target_rows_per_task) { + chunks.push_back(current_chunk); + current_chunk.clear(); + current_chunk_rows = 0; + } + + int64_t remaining_rows = target_rows_per_task - current_chunk_rows; + int64_t next_to = std::min(row_range.to, next_from + remaining_rows - 1); + current_chunk.emplace_back(next_from, next_to); + current_chunk_rows += next_to - next_from + 1; + next_from = next_to + 1; + } + } + if (!current_chunk.empty()) { + chunks.push_back(current_chunk); + } + return chunks; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup.h b/src/paimon/core/utils/blob_view_lookup.h new file mode 100644 index 00000000..169d9f83 --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup.h @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/memory/bytes.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" + +namespace paimon { + +class BatchReader; + +/// Provide a function for converting {@link BlobViewStruct}s to {@link BlobDescriptor}s by scanning +/// the upstream tables in row-range chunks. +class BlobViewLookup { + public: + using DescriptorMapping = std::unordered_map>; + /// The minimum number of rows handled by a single parallel task. + static constexpr int64_t MIN_ROW_PER_TASK = 100; + + BlobViewLookup() = delete; + ~BlobViewLookup() = delete; + + static Result CreateResolver( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool, const std::shared_ptr& executor); + + private: + class TableReadPlan { + public: + explicit TableReadPlan(const BlobViewStruct& view_struct); + + void Add(const BlobViewStruct& view_struct); + const Identifier& GetIdentifier() const; + std::vector GetFieldIds() const; + std::vector GetSortedDistinctRanges() const; + + private: + Identifier identifier_; + std::set references_by_field_id_; + std::vector row_ranges_; + }; + + static Result PreloadDescriptors( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool, const std::shared_ptr& executor); + + static Result LoadTableDescriptorChunk( + const std::shared_ptr& catalog_context, const Identifier& identifier, + const std::vector& field_ids, const std::vector& row_ranges, + const std::shared_ptr& pool); + + static Status ExtractBlobDescriptors(const Identifier& identifier, + const std::vector& field_ids, + const std::shared_ptr& pool, + BatchReader* reader, DescriptorMapping* mapping); + + static std::unordered_map GroupByIdentifier( + const std::unordered_set& view_structs); + + static int64_t TargetRowsPerTask( + const std::unordered_map& plan_by_identifier, + uint32_t thread_num); + + static std::vector> SplitRowRanges(const std::vector& row_ranges, + int64_t target_rows_per_task); + + static Result GetTableLocation( + const std::shared_ptr& catalog_context, const Identifier& identifier); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup_test.cpp b/src/paimon/core/utils/blob_view_lookup_test.cpp new file mode 100644 index 00000000..6cff2384 --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup_test.cpp @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/blob_view_lookup.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/utils/range.h" + +namespace paimon::test { + +class BlobViewLookupTest : public testing::Test { + public: + static Identifier MakeIdentifier(const std::string& database, const std::string& table) { + return Identifier(database, table); + } + + static BlobViewStruct MakeView(const std::string& database, const std::string& table, + int32_t field_id, int64_t row_id) { + return BlobViewStruct(MakeIdentifier(database, table), field_id, row_id); + } +}; + +TEST_F(BlobViewLookupTest, TestConstruct) { + BlobViewStruct view = MakeView("db", "t", /*field_id=*/7, /*row_id=*/42); + BlobViewLookup::TableReadPlan plan(view); + + ASSERT_EQ(plan.identifier_, MakeIdentifier("db", "t")); + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 1U); + auto iter = references.find(7); + ASSERT_NE(iter, references.end()); + + const auto& row_ranges = plan.row_ranges_; + ASSERT_EQ(row_ranges.size(), 1U); + ASSERT_EQ(row_ranges[0], 42); + + ASSERT_EQ(plan.GetFieldIds(), std::vector{7}); +} + +TEST_F(BlobViewLookupTest, TestAdd) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/7, /*row_id=*/10)); + + plan.Add(MakeView("db", "t", /*field_id=*/7, /*row_id=*/12)); + plan.Add(MakeView("db", "t", /*field_id=*/9, /*row_id=*/11)); + plan.Add(MakeView("db", "t", /*field_id=*/9, /*row_id=*/13)); + + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 2U); + auto iter = references.find(7); + ASSERT_NE(iter, references.end()); + iter = references.find(9); + ASSERT_NE(iter, references.end()); + + ASSERT_EQ(plan.row_ranges_, (std::vector{10, 12, 11, 13})); +} + +TEST_F(BlobViewLookupTest, TestGetFieldIds) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/3, /*row_id=*/0)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/1)); + plan.Add(MakeView("db", "t", /*field_id=*/3, /*row_id=*/2)); + plan.Add(MakeView("db", "t", /*field_id=*/2, /*row_id=*/3)); + + ASSERT_EQ(plan.GetFieldIds(), (std::vector{1, 2, 3})); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesMergesTwoAdjacentRowIds) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/6)); + + auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 1U); + ASSERT_EQ(ranges[0], Range(5, 6)); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesMergesContiguousAndGaps) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + // Out of order, with duplicates and a gap so we get two output ranges. + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/6)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/7)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/10)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/11)); + + auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 2U); + ASSERT_EQ(ranges[0], Range(5, 7)); + ASSERT_EQ(ranges[1], Range(10, 11)); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesWithNonContiguous) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/1)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/100)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/50)); + + const auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 3U); + ASSERT_EQ(ranges[0], Range(1, 1)); + ASSERT_EQ(ranges[1], Range(50, 50)); + ASSERT_EQ(ranges[2], Range(100, 100)); +} + +TEST_F(BlobViewLookupTest, TestEmptyInputProducesEmptyOutput) { + auto grouped = BlobViewLookup::GroupByIdentifier({}); + ASSERT_TRUE(grouped.empty()); +} + +TEST_F(BlobViewLookupTest, TestSingleViewStructProducesSingleGroup) { + std::unordered_set views; + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/42)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 1U); + + auto iter = grouped.find(MakeIdentifier("db", "t")); + ASSERT_NE(iter, grouped.end()); + const auto& plan = iter->second; + ASSERT_EQ(plan.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan.row_ranges_, std::vector{42}); +} + +TEST_F(BlobViewLookupTest, TestMultipleViewStructsOfSameTableAreMergedIntoOnePlan) { + std::unordered_set views; + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/2)); + views.emplace(MakeView("db", "t", /*field_id=*/4, /*row_id=*/1)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 1U); + + const auto& plan = grouped.at(MakeIdentifier("db", "t")); + ASSERT_EQ(plan.GetFieldIds(), (std::vector{3, 4})); + + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 2U); + auto iter = references.find(3); + ASSERT_NE(iter, references.end()); + iter = references.find(4); + ASSERT_NE(iter, references.end()); + + ASSERT_EQ(plan.row_ranges_.size(), 3U); +} + +TEST_F(BlobViewLookupTest, TestViewStructsOfDifferentTablesAreSplitIntoDistinctPlans) { + std::unordered_set views; + views.emplace(MakeView("db1", "t1", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db1", "t2", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db2", "t1", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db1", "t1", /*field_id=*/4, /*row_id=*/2)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 3U); + + const auto& plan_db1_t1 = grouped.at(MakeIdentifier("db1", "t1")); + ASSERT_EQ(plan_db1_t1.GetFieldIds(), (std::vector{3, 4})); + ASSERT_EQ(plan_db1_t1.row_ranges_.size(), 2U); + + const auto& plan_db1_t2 = grouped.at(MakeIdentifier("db1", "t2")); + ASSERT_EQ(plan_db1_t2.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan_db1_t2.row_ranges_.size(), 1U); + + const auto& plan_db2_t1 = grouped.at(MakeIdentifier("db2", "t1")); + ASSERT_EQ(plan_db2_t1.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan_db2_t1.row_ranges_.size(), 1U); +} + +TEST_F(BlobViewLookupTest, TestGetIdentifier) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/0)); + ASSERT_EQ(plan.GetIdentifier(), MakeIdentifier("db", "t")); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskEmptyReturnsMin) { + std::unordered_map empty; + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(empty, /*thread_num=*/100), + BlobViewLookup::MIN_ROW_PER_TASK); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskSmallTotalReturnsMin) { + std::unordered_set views; + for (int64_t row_id = 0; row_id < 10; ++row_id) { + views.emplace(MakeView("db", "t", /*field_id=*/1, row_id)); + } + auto grouped = BlobViewLookup::GroupByIdentifier(views); + // total_rows (10) is far below thread_num, so the balanced budget is clamped to + // MIN_ROW_PER_TASK. + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(grouped, /*thread_num=*/100), + BlobViewLookup::MIN_ROW_PER_TASK); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskLargeTotalBalancesAcrossThreads) { + std::unordered_set views; + const int64_t total_rows = 100001; + for (int64_t row_id = 0; row_id < total_rows; ++row_id) { + views.emplace(MakeView("db", "t", /*field_id=*/1, row_id)); + } + auto grouped = BlobViewLookup::GroupByIdentifier(views); + // ceil(100001 / 100) = 1001 + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(grouped, /*thread_num=*/100), 1001); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesEmptyInput) { + auto chunks = BlobViewLookup::SplitRowRanges({}, /*target_rows_per_task=*/10); + ASSERT_TRUE(chunks.empty()); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesSingleRangeFitsInOneChunk) { + auto chunks = BlobViewLookup::SplitRowRanges({Range(0, 4)}, /*target_rows_per_task=*/10); + ASSERT_EQ(chunks.size(), 1U); + ASSERT_EQ(chunks[0].size(), 1U); + ASSERT_EQ(chunks[0][0], Range(0, 4)); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesSplitsLargeRange) { + // [0, 9] with target 4 => [0,3], [4,7], [8,9] + auto chunks = BlobViewLookup::SplitRowRanges({Range(0, 9)}, /*target_rows_per_task=*/4); + ASSERT_EQ(chunks.size(), 3U); + ASSERT_EQ(chunks[0], (std::vector{Range(0, 3)})); + ASSERT_EQ(chunks[1], (std::vector{Range(4, 7)})); + ASSERT_EQ(chunks[2], (std::vector{Range(8, 9)})); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesPacksAcrossRanges) { + // Ranges [0,2] (3 rows) and [10,12] (3 rows) with target 4. + // chunk0 = [0,2] + part of second range [10,10] (total 4 rows) + // chunk1 = [11,12] (2 rows) + auto chunks = + BlobViewLookup::SplitRowRanges({Range(0, 2), Range(10, 12)}, /*target_rows_per_task=*/4); + ASSERT_EQ(chunks.size(), 2U); + ASSERT_EQ(chunks[0], (std::vector{Range(0, 2), Range(10, 10)})); + ASSERT_EQ(chunks[1], (std::vector{Range(11, 12)})); +} + +} // namespace paimon::test diff --git a/src/paimon/core/utils/branch_manager.cpp b/src/paimon/core/utils/branch_manager.cpp new file mode 100644 index 00000000..3ca52cf4 --- /dev/null +++ b/src/paimon/core/utils/branch_manager.cpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/branch_manager.h" + +#include +#include +#include +#include + +#include "paimon/fs/file_system.h" + +namespace paimon { + +Result> BranchManager::ListBranches(const std::shared_ptr& fs, + const std::string& table_root) { + std::vector branches = {DEFAULT_MAIN_BRANCH}; + std::string branch_dir = PathUtil::JoinPath(table_root, "branch"); + PAIMON_ASSIGN_OR_RAISE(bool is_exist, fs->Exists(branch_dir)); + if (!is_exist) { + return branches; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs->ListDir(branch_dir, &file_status_list)); + std::string branch_prefix = BRANCH_PREFIX; + for (const auto& file_status : file_status_list) { + if (!file_status->IsDir()) { + continue; + } + std::string dir_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(dir_name, branch_prefix, /*start_pos=*/0)) { + branches.push_back(dir_name.substr(branch_prefix.length())); + } + } + std::sort(branches.begin(), branches.end()); + return branches; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/branch_manager.h b/src/paimon/core/utils/branch_manager.h index ee31f738..5a4a5821 100644 --- a/src/paimon/core/utils/branch_manager.h +++ b/src/paimon/core/utils/branch_manager.h @@ -17,12 +17,20 @@ */ #pragma once +#include #include +#include #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/result.h" namespace paimon { +class FileSystem; +} // namespace paimon + +namespace paimon { +/// Utility methods for table branch paths and branch discovery. class BranchManager { public: BranchManager() = delete; @@ -31,10 +39,12 @@ class BranchManager { static constexpr char DEFAULT_MAIN_BRANCH[] = "main"; static constexpr char BRANCH_PREFIX[] = "branch-"; + /// Normalizes an empty branch name to `main`. static std::string NormalizeBranch(const std::string& branch) { return StringUtils::IsNullOrWhitespaceOnly(branch) ? DEFAULT_MAIN_BRANCH : branch; } + /// Returns the table root path for the selected branch. static std::string BranchPath(const std::string& table_root, const std::string& branch) { return IsMainBranch(branch) ? table_root @@ -42,8 +52,13 @@ class BranchManager { "/branch/" + std::string(BRANCH_PREFIX) + branch); } + /// Returns whether the branch is the default main branch. static bool IsMainBranch(const std::string& branch) { return branch == DEFAULT_MAIN_BRANCH; } + + /// Lists all branches for a table, including `main`. + static Result> ListBranches(const std::shared_ptr& fs, + const std::string& table_root); }; } // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager.cpp b/src/paimon/core/utils/consumer_manager.cpp new file mode 100644 index 00000000..4bde69b1 --- /dev/null +++ b/src/paimon/core/utils/consumer_manager.cpp @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/consumer_manager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "rapidjson/document.h" + +namespace paimon { + +ConsumerManager::ConsumerManager(std::shared_ptr fs, std::string table_path, + std::string branch) + : fs_(std::move(fs)), + table_path_(std::move(table_path)), + branch_(BranchManager::NormalizeBranch(branch)) {} + +std::string ConsumerManager::ConsumerDirectory() const { + return PathUtil::JoinPath(BranchManager::BranchPath(table_path_, branch_), "consumer"); +} + +std::string ConsumerManager::ConsumerPath(const std::string& consumer_id) const { + return PathUtil::JoinPath(ConsumerDirectory(), std::string(kConsumerPrefix) + consumer_id); +} + +Result> ConsumerManager::ListConsumers() const { + std::vector consumers; + std::string consumer_dir = ConsumerDirectory(); + PAIMON_ASSIGN_OR_RAISE(bool exists, fs_->Exists(consumer_dir)); + if (!exists) { + return consumers; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs_->ListDir(consumer_dir, &file_status_list)); + std::string prefix = kConsumerPrefix; + for (const auto& file_status : file_status_list) { + if (file_status->IsDir()) { + continue; + } + std::string file_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(file_name, prefix, /*start_pos=*/0)) { + consumers.push_back(file_name.substr(prefix.length())); + } + } + std::sort(consumers.begin(), consumers.end()); + return consumers; +} + +Result> ConsumerManager::GetNextSnapshotId( + const std::string& consumer_id) const { + constexpr int32_t kMaxRetryCount = 10; + constexpr int32_t kRetryIntervalMillis = 200; + Status last_error; + for (int32_t i = 0; i < kMaxRetryCount; ++i) { + std::string content; + Status read_status = fs_->ReadFile(ConsumerPath(consumer_id), &content); + if (!read_status.ok()) { + if (read_status.IsNotExist()) { + return std::optional(); + } + return read_status; + } + std::optional snapshot_id = StringUtils::StringToValue(content); + if (snapshot_id) { + return snapshot_id; + } + + rapidjson::Document document; + document.Parse(content.c_str()); + if (!document.HasParseError() && document.IsObject() && + document.HasMember("nextSnapshot") && document["nextSnapshot"].IsInt64()) { + return std::optional(document["nextSnapshot"].GetInt64()); + } + + last_error = + Status::Invalid("failed to parse consumer metadata: ", ConsumerPath(consumer_id)); + std::this_thread::sleep_for(std::chrono::milliseconds(kRetryIntervalMillis)); + } + return last_error; +} + +Result> ConsumerManager::Consumers() const { + std::map consumers; + PAIMON_ASSIGN_OR_RAISE(std::vector consumer_ids, ListConsumers()); + for (const auto& id : consumer_ids) { + PAIMON_ASSIGN_OR_RAISE(std::optional next_snapshot_id, GetNextSnapshotId(id)); + if (next_snapshot_id) { + consumers[id] = next_snapshot_id.value(); + } + } + return consumers; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager.h b/src/paimon/core/utils/consumer_manager.h new file mode 100644 index 00000000..de473fd2 --- /dev/null +++ b/src/paimon/core/utils/consumer_manager.h @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +class FileSystem; + +/// Manager for table streaming consumer metadata files. +/// +/// Consumers are stored under the selected table branch as `consumer/consumer-*` files. +class ConsumerManager { + public: + /// File name prefix for persisted consumer state files. + static constexpr char kConsumerPrefix[] = "consumer-"; + + ConsumerManager(std::shared_ptr fs, std::string table_path, std::string branch); + + /// Returns the consumer metadata directory for the selected table branch. + std::string ConsumerDirectory() const; + + /// Returns the metadata file path for a specific consumer id. + std::string ConsumerPath(const std::string& consumer_id) const; + + /// Lists consumer ids found in the consumer metadata directory. + Result> ListConsumers() const; + + /// Reads the next snapshot id persisted for the given consumer id. + Result> GetNextSnapshotId(const std::string& consumer_id) const; + + /// Reads all consumers and their next snapshot ids. + Result> Consumers() const; + + private: + std::shared_ptr fs_; + std::string table_path_; + std::string branch_; +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager_test.cpp b/src/paimon/core/utils/consumer_manager_test.cpp new file mode 100644 index 00000000..ae9252ba --- /dev/null +++ b/src/paimon/core/utils/consumer_manager_test.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/consumer_manager.h" + +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(ConsumerManagerTest, TestBranchConsumerPath) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string consumer_dir = + PathUtil::JoinPath(BranchManager::BranchPath(table_path, "dev"), "consumer"); + ASSERT_OK(fs->Mkdirs(consumer_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(consumer_dir, "consumer-c1"), + R"({"nextSnapshot":42})", /*overwrite=*/true)); + + ConsumerManager manager(fs, table_path, "dev"); + ASSERT_EQ(manager.ConsumerDirectory(), consumer_dir); + ASSERT_OK_AND_ASSIGN(std::vector consumers, manager.ListConsumers()); + ASSERT_EQ(consumers, (std::vector{"c1"})); + ASSERT_OK_AND_ASSIGN(std::optional next_snapshot_id, manager.GetNextSnapshotId("c1")); + ASSERT_EQ(next_snapshot_id, 42); + ASSERT_OK_AND_ASSIGN(auto all_consumers, manager.Consumers()); + ASSERT_EQ(all_consumers, (std::map{{"c1", 42}})); +} + +} // namespace paimon::test diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index e12b82d4..df3791e3 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -19,10 +19,8 @@ #include "paimon/core/utils/field_mapping.h" -#include -#include #include -#include +#include #include "arrow/type.h" #include "fmt/format.h" @@ -32,6 +30,7 @@ #include "paimon/common/utils/object_utils.h" #include "paimon/core/casting/cast_executor_factory.h" #include "paimon/core/casting/casting_utils.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/defs.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" @@ -77,8 +76,8 @@ Result> FieldMappingBuilder::CreateFieldMapping( // generate non-exist field info std::optional non_exist_field_info = CreateNonExistFieldInfo(data_fields); - // generate exist field info - ExistFieldInfo exist_field_info = CreateExistFieldInfo(data_fields); + // generate exist field info (includes nested type pruning) + PAIMON_ASSIGN_OR_RAISE(ExistFieldInfo exist_field_info, CreateExistFieldInfo(data_fields)); // key: partition key, value: partition idx std::map partition_key_to_idx = @@ -92,7 +91,7 @@ Result> FieldMappingBuilder::CreateFieldMapping( return std::make_unique(partition_info, non_partition_info, non_exist_field_info); } -ExistFieldInfo FieldMappingBuilder::CreateExistFieldInfo( +Result FieldMappingBuilder::CreateExistFieldInfo( const std::vector& data_fields) const { // key:field id, value: {target_idx, read field} std::map> field_id_to_read_fields; @@ -106,8 +105,22 @@ ExistFieldInfo FieldMappingBuilder::CreateExistFieldInfo( auto iter = field_id_to_read_fields.find(data_field.Id()); if (iter != field_id_to_read_fields.end()) { const auto& [target_idx, read_field] = iter->second; + + // Recursively prune nested types in data_field to match read_field's + // projection. For atomic types this is a no-op. + PAIMON_ASSIGN_OR_RAISE( + std::optional> pruned_type, + NestedProjectionUtils::PruneDataType(read_field.Type(), data_field.Type())); + if (!pruned_type.has_value()) { + // All sub-fields pruned away — treat as non-existent. + continue; + } + + DataField pruned_data_field(data_field.Id(), + data_field.ArrowField()->WithType(pruned_type.value()), + data_field.Description()); exist_field_info.exist_read_schema.push_back(read_field); - exist_field_info.exist_data_schema.push_back(data_field); + exist_field_info.exist_data_schema.push_back(pruned_data_field); exist_field_info.idx_in_target_read_schema.push_back(target_idx); } } @@ -129,6 +142,19 @@ std::optional FieldMappingBuilder::CreateNonExistFieldInfo( if (iter == field_id_to_data_fields.end()) { non_exist_field_info.non_exist_read_schema.push_back(read_field); non_exist_field_info.idx_in_target_read_schema.push_back(i); + continue; + } + + // Empty STRUCT projection (f1: struct<>) is a valid request, but + // cannot be read from data files directly. Materialize it as nulls. + if (read_field.Type()->id() == arrow::Type::STRUCT && + iter->second.Type()->id() == arrow::Type::STRUCT) { + auto read_struct = std::static_pointer_cast(read_field.Type()); + auto data_struct = std::static_pointer_cast(iter->second.Type()); + if (read_struct->num_fields() == 0 && data_struct->num_fields() > 0) { + non_exist_field_info.non_exist_read_schema.push_back(read_field); + non_exist_field_info.idx_in_target_read_schema.push_back(i); + } } } if (non_exist_field_info.idx_in_target_read_schema.empty()) { @@ -149,9 +175,13 @@ Result>> FieldMappingBuilder::CreateDa FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { - if (read_type == FieldType::MAP || read_type == FieldType::ARRAY || - read_type == FieldType::STRUCT) { - return Status::Invalid("Only support column type evolution in atomic data type."); + auto read_type_id = read_fields[i].Type()->id(); + if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || + read_type_id == arrow::Type::MAP) { + // Nested type differs by pruning/evolution; the reader's reshape + // handles it, no scalar cast. + cast_executors.push_back(nullptr); + continue; } auto executor_factory = CastExecutorFactory::GetCastExecutorFactory(); auto cast_executor = @@ -175,13 +205,16 @@ Result FieldMappingBuilder::CreateNonPartitionInfo( const std::vector& data_fields, const ExistFieldInfo& exist_field_info, const std::map& partition_keys) const { NonPartitionInfo non_partition_info; + const std::vector propagated_metadata_keys = {DataField::MAP_SELECTED_KEYS}; for (size_t i = 0; i < exist_field_info.exist_data_schema.size(); i++) { const auto& data_field = exist_field_info.exist_data_schema[i]; const auto& read_field = exist_field_info.exist_read_schema[i]; auto iter = partition_keys.find(read_field.Name()); if (iter == partition_keys.end()) { non_partition_info.non_partition_read_schema.push_back(read_field); - non_partition_info.non_partition_data_schema.push_back(data_field); + non_partition_info.non_partition_data_schema.push_back( + DataField::MergeFieldMetadataByWhitelist(data_field, read_field, + propagated_metadata_keys)); non_partition_info.idx_in_target_read_schema.push_back( exist_field_info.idx_in_target_read_schema[i]); } diff --git a/src/paimon/core/utils/field_mapping.h b/src/paimon/core/utils/field_mapping.h index ae7795c5..55b89253 100644 --- a/src/paimon/core/utils/field_mapping.h +++ b/src/paimon/core/utils/field_mapping.h @@ -18,8 +18,6 @@ */ #pragma once -#include -#include #include #include #include @@ -85,7 +83,7 @@ class FieldMappingBuilder { std::optional CreateNonExistFieldInfo( const std::vector& data_fields) const; - ExistFieldInfo CreateExistFieldInfo(const std::vector& data_fields) const; + Result CreateExistFieldInfo(const std::vector& data_fields) const; Result CreateNonPartitionInfo( const std::vector& data_fields, const ExistFieldInfo& exist_field_info, diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index 153932df..2de53d25 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -645,4 +645,35 @@ TEST_F(FieldMappingTest, TestCompoundPredicateWithoutPushDown) { CheckNonPartitionInfo(mapping->non_partition_info, expected_non_part_info); } +TEST_F(FieldMappingTest, TestMapSelectedKeysMetadataPropagatedToDataSchema) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto selected_keys_metadata = arrow::KeyValueMetadata::Make( + {DataField::MAP_SELECTED_KEYS, "custom.key"}, {"k1,k2", "should_not_propagate"}); + + std::vector read_fields = { + DataField(0, arrow::field("m", map_type)->WithMetadata(selected_keys_metadata)), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + std::vector data_fields = { + DataField(0, arrow::field("m", map_type)), + }; + + ASSERT_OK_AND_ASSIGN( + auto mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields)); + + ASSERT_EQ(mapping->non_partition_info.non_partition_data_schema.size(), 1); + auto propagated_field = mapping->non_partition_info.non_partition_data_schema[0].ArrowField(); + ASSERT_TRUE(propagated_field->HasMetadata()); + ASSERT_TRUE(propagated_field->metadata()); + auto selected_keys_result = propagated_field->metadata()->Get(DataField::MAP_SELECTED_KEYS); + ASSERT_TRUE(selected_keys_result.ok()); + std::string selected_keys = selected_keys_result.ValueUnsafe(); + ASSERT_EQ(selected_keys, "k1,k2"); + auto custom_metadata_result = propagated_field->metadata()->Get("custom.key"); + ASSERT_FALSE(custom_metadata_result.ok()); +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/file_store_path_factory.cpp b/src/paimon/core/utils/file_store_path_factory.cpp index d17b223f..ef4aae42 100644 --- a/src/paimon/core/utils/file_store_path_factory.cpp +++ b/src/paimon/core/utils/file_store_path_factory.cpp @@ -21,7 +21,6 @@ #include #include "paimon/common/fs/external_path_provider.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/uuid.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/index/index_in_data_file_dir_path_factory.h" @@ -30,6 +29,7 @@ #include "paimon/core/utils/partition_path_utils.h" #include "paimon/core/utils/path_factory.h" #include "paimon/macros.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" namespace arrow { diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp new file mode 100644 index 00000000..97c08e9a --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -0,0 +1,720 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/nested_projection_utils.h" + +#include +#include +#include +#include +#include + +#include "arrow/array/array_nested.h" +#include "arrow/array/array_primitive.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/array/concatenate.h" +#include "arrow/array/util.h" +#include "arrow/compute/cast.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/casting/casting_utils.h" +#include "paimon/status.h" + +namespace paimon { + +std::shared_ptr NestedProjectionUtils::FindFieldByName( + const arrow::FieldVector& fields, const std::string& name) { + for (const auto& field : fields) { + if (field->name() == name) { + return field; + } + } + return nullptr; +} + +Result NestedProjectionUtils::GetPaimonFieldId( + const std::shared_ptr& field) { + if (!field->HasMetadata() || !field->metadata()) { + return Status::Invalid(fmt::format( + "GetPaimonFieldId failed, do not exist metadata in field {}", field->name())); + } + auto result = field->metadata()->Get(DataField::FIELD_ID); + if (!result.ok()) { + return Status::Invalid( + fmt::format("GetPaimonFieldId failed, cannot find field_id in metadata in field {}", + field->name())); + } + std::optional field_id = StringUtils::StringToValue(result.ValueUnsafe()); + if (!field_id) { + return Status::Invalid( + fmt::format("GetPaimonFieldId failed, cannot find convert field_id {} to int32", + result.ValueUnsafe())); + } + return field_id.value(); +} + +Result> NestedProjectionUtils::FindFieldByPaimonId( + const std::shared_ptr& struct_type, int32_t field_id) { + for (const auto& child : struct_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t paimon_field_id, GetPaimonFieldId(child)); + if (paimon_field_id == field_id) { + return child; + } + } + return Status::Invalid( + fmt::format("cannot find field {} in struct type {}", field_id, struct_type->ToString())); +} + +Result NestedProjectionUtils::HasNestedSubfieldProjectionType( + const std::shared_ptr& file_type, + const std::shared_ptr& read_type) { + switch (file_type->id()) { + case arrow::Type::STRUCT: { + if (VariantAccessUtils::IsVariantAccessType(read_type)) { + // A variant-access projection is resolved by the variant read plans, not by + // nested subfield projection. + return false; + } + if (read_type->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_struct = std::static_pointer_cast(file_type); + auto read_struct = std::static_pointer_cast(read_type); + bool field_count_diff = read_struct->num_fields() != file_struct->num_fields(); + for (const auto& read_child : read_struct->fields()) { + auto file_child = FindFieldByName(file_struct->fields(), read_child->name()); + if (!file_child) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType found requested struct child '{}' " + "missing in file type {}", + read_child->name(), file_type->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + bool child_has_nested_projection, + HasNestedSubfieldProjectionType(file_child->type(), read_child->type())); + if (child_has_nested_projection) { + return true; + } + } + return field_count_diff; + } + case arrow::Type::LIST: { + if (read_type->id() != arrow::Type::LIST) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_list = std::static_pointer_cast(file_type); + auto read_list = std::static_pointer_cast(read_type); + return HasNestedSubfieldProjectionType(file_list->value_type(), + read_list->value_type()); + } + case arrow::Type::MAP: { + if (read_type->id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_map = std::static_pointer_cast(file_type); + auto read_map = std::static_pointer_cast(read_type); + PAIMON_ASSIGN_OR_RAISE( + bool key_has_nested_projection, + HasNestedSubfieldProjectionType(file_map->key_type(), read_map->key_type())); + if (key_has_nested_projection) { + return true; + } + return HasNestedSubfieldProjectionType(file_map->item_type(), read_map->item_type()); + } + default: + return false; + } +} + +namespace { + +// Structural equality that also compares paimon field IDs on STRUCT children, so a +// drop+add of a same-name/same-type field (new ID) is not treated as a no-op. +Result EqualWithFieldIds(const std::shared_ptr& a, + const std::shared_ptr& b) { + if (a->id() != b->id() || a->num_fields() != b->num_fields()) { + return false; + } + if (a->num_fields() == 0) { + return a->Equals(*b); + } + for (int32_t i = 0; i < a->num_fields(); ++i) { + const auto& fa = a->field(i); + const auto& fb = b->field(i); + if (fa->nullable() != fb->nullable()) { + return false; + } + if (a->id() == arrow::Type::STRUCT) { + if (fa->name() != fb->name()) { + return false; + } + // Compare IDs only when present (a map entry's key/value carry none). + auto id_a = NestedProjectionUtils::GetPaimonFieldId(fa); + auto id_b = NestedProjectionUtils::GetPaimonFieldId(fb); + if (id_a.ok() && id_b.ok() && id_a.value() != id_b.value()) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(bool child_equal, EqualWithFieldIds(fa->type(), fb->type())); + if (!child_equal) { + return false; + } + } + return true; +} + +/// Whether `read_type` is `data_type` with variant columns replaced by their variant-access +/// projections and nothing else changed (matching paimon field IDs). +Result IsVariantAccessSubstitution(const std::shared_ptr& read_type, + const std::shared_ptr& data_type) { + PAIMON_ASSIGN_OR_RAISE(bool equal, EqualWithFieldIds(read_type, data_type)); + if (equal) { + return true; + } + if (VariantAccessUtils::IsVariantAccessType(read_type) && + VariantTypeUtils::IsUnshreddedVariantType(data_type)) { + return true; + } + // Any other difference in shape, including a dropped field, is a real projection. + if (read_type->id() != data_type->id() || read_type->num_fields() != data_type->num_fields()) { + return false; + } + for (int32_t i = 0; i < read_type->num_fields(); ++i) { + const std::shared_ptr& read_child = read_type->field(i); + const std::shared_ptr& data_child = data_type->field(i); + // LIST and MAP name their children by format convention, so only STRUCT is + // matched by name and field ID. + if (read_type->id() == arrow::Type::STRUCT) { + if (read_child->name() != data_child->name()) { + return false; + } + auto id_r = NestedProjectionUtils::GetPaimonFieldId(read_child); + auto id_d = NestedProjectionUtils::GetPaimonFieldId(data_child); + if (id_r.ok() && id_d.ok() && id_r.value() != id_d.value()) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(bool sub, + IsVariantAccessSubstitution(read_child->type(), data_child->type())); + if (!sub) { + return false; + } + } + return true; +} + +// Reconcile a LIST/MAP item: read may ADD fields (evolution, null-filled +// downstream) but must not DROP one. Returns the file-readable item type; +// `container` names the container ("list"/"map") for the error message. +Result> PruneRepeatedItemType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type, const char* container) { + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type)); + if (same) { + return data_type; + } + PAIMON_ASSIGN_OR_RAISE(bool substitution, IsVariantAccessSubstitution(read_type, data_type)); + if (substitution) { + return read_type; + } + if (read_type->id() != data_type->id()) { + return Status::Invalid( + fmt::format("PruneDataType nested item type mismatch inside {}: read {} vs data {}", + container, read_type->ToString(), data_type->ToString())); + } + switch (data_type->id()) { + case arrow::Type::STRUCT: { + arrow::FieldVector item_fields; + for (const auto& data_child : data_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t data_child_id, + NestedProjectionUtils::GetPaimonFieldId(data_child)); + std::shared_ptr read_child; + for (const auto& candidate : read_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id, + NestedProjectionUtils::GetPaimonFieldId(candidate)); + if (candidate_id == data_child_id) { + read_child = candidate; + break; + } + } + if (!read_child) { + // A file field is dropped -- a real partial projection. + return Status::Invalid(fmt::format( + "PruneDataType does not support partial projection inside {}: src {} vs " + "target {}", + container, data_type->ToString(), read_type->ToString())); + } + if (read_child->name() != data_child->name()) { + return Status::Invalid(fmt::format( + "PruneDataType does not support renaming inside {}: field id {} read '{}' " + "vs data '{}'", + container, data_child_id, read_child->name(), data_child->name())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item_child_type, + PruneRepeatedItemType(read_child->type(), data_child->type(), container)); + item_fields.push_back(data_child->WithType(item_child_type)); + } + return arrow::struct_(item_fields); + } + case arrow::Type::LIST: { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item, + PruneRepeatedItemType(read_type->field(0)->type(), + data_type->field(0)->type(), container)); + return arrow::list(data_type->field(0)->WithType(item)); + } + case arrow::Type::MAP: { + auto read_map = std::static_pointer_cast(read_type); + auto data_map = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key, + PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), container)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item, + PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), container)); + return std::static_pointer_cast(std::make_shared( + data_map->key_field()->WithType(key), data_map->item_field()->WithType(item), + data_map->keys_sorted())); + } + default: + return data_type; + } +} + +} // namespace + +Result>> NestedProjectionUtils::PruneDataType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type) { + // Identical types (including paimon field IDs) need no pruning. + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type)); + if (same) { + return std::optional>(data_type); + } + + switch (read_type->id()) { + case arrow::Type::STRUCT: { + if (VariantAccessUtils::IsVariantAccessType(read_type) && + VariantTypeUtils::IsUnshreddedVariantType(data_type)) { + // A variant-access projection replaces the variant column type; pass it through + // so the read path extracts the described paths. + return std::optional>(read_type); + } + arrow::FieldVector pruned_fields; + for (const auto& read_child : read_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(int32_t read_child_id, GetPaimonFieldId(read_child)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_child, + FindFieldByPaimonId(data_type, read_child_id)); + if (read_child->name() != data_child->name()) { + return Status::Invalid(fmt::format( + "PruneDataType does not support schema evolution inside struct: nested " + "field id {} name mismatch: read '{}' vs data '{}'", + read_child_id, read_child->name(), data_child->name())); + } + if (read_child->type()->id() != data_child->type()->id()) { + return Status::Invalid(fmt::format( + "PruneDataType nested field type mismatch for '{}': read {} vs data {}", + read_child->name(), read_child->type()->ToString(), + data_child->type()->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::optional> pruned_child_type, + PruneDataType(read_child->type(), data_child->type())); + if (!pruned_child_type.has_value()) { + // All sub-fields of this child were pruned away; skip it. + continue; + } + pruned_fields.push_back(data_child->WithType(pruned_child_type.value())); + } + if (pruned_fields.empty()) { + // All fields pruned — return nullopt so the caller can skip this field. + return std::optional>(std::nullopt); + } + return std::optional>(arrow::struct_(pruned_fields)); + } + case arrow::Type::LIST: { + PAIMON_ASSIGN_OR_RAISE(bool list_substitution, + IsVariantAccessSubstitution(read_type, data_type)); + if (list_substitution) { + return std::optional>(read_type); + } + // Added fields (schema evolution) are allowed; dropped fields still fail. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item, + PruneRepeatedItemType(read_type->field(0)->type(), + data_type->field(0)->type(), "list")); + return std::optional>( + arrow::list(data_type->field(0)->WithType(item))); + } + case arrow::Type::MAP: { + PAIMON_ASSIGN_OR_RAISE(bool map_substitution, + IsVariantAccessSubstitution(read_type, data_type)); + if (map_substitution) { + return std::optional>(read_type); + } + auto read_map = std::static_pointer_cast(read_type); + auto data_map = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr key, + PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), "map")); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr item, + PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), "map")); + return std::optional>(std::make_shared( + data_map->key_field()->WithType(key), data_map->item_field()->WithType(item), + data_map->keys_sorted())); + } + default: + // Atomic type: return data_type as-is (type evolution is handled + // separately by CastExecutor). + return std::optional>(data_type); + } +} + +Result NestedProjectionUtils::HasNestedSubfieldProjection( + const std::shared_ptr& file_schema, + const std::shared_ptr& read_schema) { + for (const auto& read_field : read_schema->fields()) { + auto file_field = file_schema->GetFieldByName(read_field->name()); + if (!file_field) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjection found read field '{}' missing in file schema {}", + read_field->name(), file_schema->ToString())); + } + if (read_field->type()->id() == arrow::Type::STRUCT || + read_field->type()->id() == arrow::Type::LIST || + read_field->type()->id() == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE( + bool has_nested_projection, + HasNestedSubfieldProjectionType(file_field->type(), read_field->type())); + if (has_nested_projection) { + return true; + } + } + } + return false; +} + +// Map selected-keys support +Result> NestedProjectionUtils::GetMapSelectedKeys( + const std::shared_ptr& field) { + std::vector result; + if (!field->HasMetadata() || !field->metadata()) { + return result; + } + auto get_result = field->metadata()->Get(DataField::MAP_SELECTED_KEYS); + if (!get_result.ok()) { + return result; + } + auto tokens = StringUtils::Split(get_result.ValueUnsafe(), ",", /*ignore_empty=*/false); + std::unordered_set deduplicated; + deduplicated.reserve(tokens.size()); + for (const auto& token : tokens) { + if (!deduplicated.insert(token).second) { + return Status::Invalid(fmt::format("Duplicate selected key '{}' in {} metadata", token, + DataField::MAP_SELECTED_KEYS)); + } + result.push_back(token); + } + return result; +} + +namespace { + +struct MapKeyAccessor { + std::shared_ptr string_keys; + std::shared_ptr dict_keys; + std::shared_ptr dict_values; + std::shared_ptr dict_large_values; +}; + +Result BuildMapKeyAccessor(const std::shared_ptr& key_array) { + MapKeyAccessor accessor; + if (key_array->type_id() == arrow::Type::STRING) { + accessor.string_keys = std::static_pointer_cast(key_array); + return accessor; + } + if (key_array->type_id() == arrow::Type::DICTIONARY) { + auto dict_type = std::static_pointer_cast(key_array->type()); + if (dict_type->value_type()->id() != arrow::Type::STRING && + dict_type->value_type()->id() != arrow::Type::LARGE_STRING) { + return Status::Invalid( + fmt::format("FilterMapArrayBySelectedKeys only supports string keys or " + "dictionary keys, got {}", + key_array->type()->ToString())); + } + accessor.dict_keys = std::static_pointer_cast(key_array); + if (dict_type->value_type()->id() == arrow::Type::STRING) { + accessor.dict_values = + std::static_pointer_cast(accessor.dict_keys->dictionary()); + } else { + accessor.dict_large_values = + std::static_pointer_cast(accessor.dict_keys->dictionary()); + } + return accessor; + } + return Status::Invalid( + fmt::format("FilterMapArrayBySelectedKeys only supports string keys or " + "dictionary keys, got {}", + key_array->type()->ToString())); +} + +Result GetMapKeyViewAt(const MapKeyAccessor& accessor, int64_t entry_idx) { + if (accessor.string_keys) { + if (accessor.string_keys->IsNull(entry_idx)) { + return Status::Invalid("FilterMapArrayBySelectedKeys found null map key at entry " + + std::to_string(entry_idx)); + } + return accessor.string_keys->GetView(entry_idx); + } + + if (accessor.dict_keys->IsNull(entry_idx)) { + return Status::Invalid("FilterMapArrayBySelectedKeys found null map key at entry " + + std::to_string(entry_idx)); + } + int64_t dict_idx = accessor.dict_keys->GetValueIndex(entry_idx); + if (accessor.dict_values) { + if (accessor.dict_values->IsNull(dict_idx)) { + return Status::Invalid( + "FilterMapArrayBySelectedKeys found null dictionary map key at dictionary index " + + std::to_string(dict_idx)); + } + return accessor.dict_values->GetView(dict_idx); + } + + if (accessor.dict_large_values->IsNull(dict_idx)) { + return Status::Invalid( + "FilterMapArrayBySelectedKeys found null dictionary map key at dictionary index " + + std::to_string(dict_idx)); + } + return accessor.dict_large_values->GetView(dict_idx); +} + +} // namespace + +Result> NestedProjectionUtils::FilterMapArrayBySelectedKeys( + const std::shared_ptr& array, const std::vector& selected_keys, + arrow::MemoryPool* pool) { + if (selected_keys.empty() || !array || array->length() == 0) { + return array; + } + if (pool == nullptr) { + return Status::Invalid("FilterMapArrayBySelectedKeys requires a non-null memory pool"); + } + + if (array->type_id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format( + "FilterMapArrayBySelectedKeys requires map array, got {}", array->type()->ToString())); + } + + auto map_array = std::static_pointer_cast(array); + auto map_type = std::static_pointer_cast(array->type()); + assert(map_array && map_type); + + auto key_array = map_array->keys(); + PAIMON_ASSIGN_OR_RAISE(MapKeyAccessor key_accessor, BuildMapKeyAccessor(key_array)); + + auto values_array = map_array->items(); + int64_t num_maps = map_array->length(); + + std::unordered_set deduplicated; + deduplicated.reserve(selected_keys.size()); + for (const auto& selected_key : selected_keys) { + if (!deduplicated.insert(selected_key).second) { + return Status::Invalid(fmt::format("Duplicate selected key '{}' in {} metadata", + selected_key, DataField::MAP_SELECTED_KEYS)); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_builder_u, + arrow::MakeBuilder(arrow::utf8(), pool)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr value_builder_u, + arrow::MakeBuilder(values_array->type(), pool)); + arrow::MapBuilder map_builder(pool, std::move(key_builder_u), std::move(value_builder_u)); + auto* key_builder = static_cast(map_builder.key_builder()); + auto* value_builder = map_builder.item_builder(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Reserve(num_maps)); + + for (int64_t map_idx = 0; map_idx < num_maps; ++map_idx) { + if (map_array->IsNull(map_idx)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append()); + int64_t start = map_array->value_offset(map_idx); + int64_t end = map_array->value_offset(map_idx + 1); + + // Keep selected keys in the exact selected_keys order. + for (const auto& selected_key : selected_keys) { + for (int64_t entry_idx = start; entry_idx < end; ++entry_idx) { + PAIMON_ASSIGN_OR_RAISE(std::string_view key_view, + GetMapKeyViewAt(key_accessor, entry_idx)); + if (key_view == selected_key) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append( + key_view.data(), static_cast(key_view.size()))); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + value_builder->AppendArraySlice(*values_array->data(), entry_idx, 1)); + } + } + } + } + + std::shared_ptr result_map; + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Finish(&result_map)); + return result_map; +} + +namespace { +// Strips physical-only differences from a leaf type: ORC lazy decoding wraps +// strings in a dictionary and may widen them to large_string. binary is not +// dictionary-encoded and large_binary is blob's real type, so neither is +// normalized. Two leaves with equal normalized types hold the same logical +// values. +std::shared_ptr NormalizeLeafRepresentation( + const std::shared_ptr& type) { + auto t = type; + if (t->id() == arrow::Type::DICTIONARY) { + t = std::static_pointer_cast(t)->value_type(); + } + if (t->id() == arrow::Type::LARGE_STRING) { + return arrow::utf8(); + } + return t; +} +} // namespace + +Result> NestedProjectionUtils::AlignArrayToReadType( + const std::shared_ptr& array, const std::shared_ptr& read_type, + arrow::MemoryPool* pool) { + PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(array->type(), read_type)); + if (same) { + return array; + } + // Produce exactly `read_type` so every file yields the same output type: rebuild + // STRUCT/LIST/MAP with read-side types/nullability and cast a leaf (decodes dict). + const auto& data = array->data(); + switch (read_type->id()) { + case arrow::Type::STRUCT: { + if (array->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } + const auto& array_type = array->type(); + std::vector> children; + children.reserve(read_type->num_fields()); + for (const auto& read_field : read_type->fields()) { + // Match by name (parquet drops nested field-id metadata); if both + // carry IDs they must agree, so a drop+add same-name field won't match. + auto read_id = GetPaimonFieldId(read_field); + int32_t match = -1; + for (int32_t j = 0; j < array_type->num_fields(); j++) { + const auto& array_field = array_type->field(j); + if (array_field->name() != read_field->name()) { + continue; + } + auto data_id = GetPaimonFieldId(array_field); + if (read_id.ok() && data_id.ok() && read_id.value() != data_id.value()) { + continue; + } + match = j; + break; + } + if (match >= 0) { + auto child = arrow::MakeArray(data->child_data[match]); + PAIMON_ASSIGN_OR_RAISE(child, + AlignArrayToReadType(child, read_field->type(), pool)); + children.push_back(child->data()); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_child, + arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length, + pool)); + children.push_back(null_child->data()); + } + } + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = std::move(children); + return arrow::MakeArray(new_data); + } + case arrow::Type::LIST: { + if (array->type()->id() != arrow::Type::LIST) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } + auto read_list = std::static_pointer_cast(read_type); + auto values = arrow::MakeArray(data->child_data[0]); + PAIMON_ASSIGN_OR_RAISE(values, + AlignArrayToReadType(values, read_list->value_type(), pool)); + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {values->data()}; + return arrow::MakeArray(new_data); + } + case arrow::Type::MAP: { + if (array->type()->id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}", + array->type()->ToString(), + read_type->ToString())); + } + auto read_map = std::static_pointer_cast(read_type); + const auto& entries_data = data->child_data[0]; + auto key = arrow::MakeArray(entries_data->child_data[0]); + auto value = arrow::MakeArray(entries_data->child_data[1]); + PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool)); + PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool)); + auto new_entries = entries_data->Copy(); + new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()}); + new_entries->child_data = {key->data(), value->data()}; + auto new_data = data->Copy(); + new_data->type = read_type; + new_data->child_data = {new_entries}; + return arrow::MakeArray(new_data); + } + default: { + // Leaf: only physical-representation differences are valid here (ORC + // dictionary encoding, string/binary offset width). Genuine type + // evolution is handled by FieldMappingReader's cast executors and + // rejected upstream in PruneDataType, so fail anything else. + if (!NormalizeLeafRepresentation(array->type()) + ->Equals(*NormalizeLeafRepresentation(read_type))) { + return Status::Invalid( + fmt::format("AlignArrayToReadType unsupported leaf type change: data {} vs " + "read {}", + array->type()->ToString(), read_type->ToString())); + } + return CastingUtils::Cast(array, read_type, arrow::compute::CastOptions::Safe(), pool); + } + } +} + +} // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h new file mode 100644 index 00000000..b0ab8fc2 --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/type.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/result.h" + +namespace paimon { + +/// Utility class for nested column pruning and map key selection. +class PAIMON_EXPORT NestedProjectionUtils { + public: + NestedProjectionUtils() = delete; + ~NestedProjectionUtils() = delete; + + static std::shared_ptr FindFieldByName(const arrow::FieldVector& fields, + const std::string& name); + + /// Extract the paimon field ID from an Arrow field's metadata ("paimon.id"). + /// @return The paimon.id in metadata, or return bad status if the metadata key is not present + /// or convert error. + static Result GetPaimonFieldId(const std::shared_ptr& field); + + /// Find a child field in a STRUCT DataType by paimon field ID. + /// @return The specific arrow field, or return bad status if no child has the given ID. + static Result> FindFieldByPaimonId( + const std::shared_ptr& struct_type, int32_t field_id); + + /// Recursively prune `data_type` so that only the sub-fields requested by + /// `read_type` are retained. Matching is done by paimon field ID to support + /// schema evolution (field renames). + /// + /// Supported nesting: STRUCT, LIST (element recurse), MAP (key/value recurse). + /// For atomic types, `data_type` is returned as-is. + /// + /// @return std::nullopt when all sub-fields of a STRUCT are pruned away + /// (caller should skip this field entirely, mirroring Java's null return). + static Result>> PruneDataType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type); + + /// @return true if `read_schema` requests a nested sub-field projection against + /// `file_schema` (same top-level field, but nested STRUCT/LIST/MAP subtree is pruned). + static Result HasNestedSubfieldProjection( + const std::shared_ptr& file_schema, + const std::shared_ptr& read_schema); + + /// Parse the "paimon.map.selected-keys" metadata from an Arrow field. + /// @return an empty vector if has no metadata, or the specific metadata key is absent. + /// The metadata value is a comma-separated string, e.g. "key1,key2". + /// Empty tokens are preserved ("" means selecting empty-string keys), and duplicate + /// selected keys are rejected as invalid. + static Result> GetMapSelectedKeys( + const std::shared_ptr& field); + + /// Filter a MapArray so that only entries whose key is in `selected_keys` are kept. + /// Supports string keys and dictionary keys. + /// The output map entry order follows + /// `selected_keys` order, and duplicate selected keys are rejected. + /// @return the original array unchanged if `selected_keys` is empty. + static Result> FilterMapArrayBySelectedKeys( + const std::shared_ptr& map_array, + const std::vector& selected_keys, arrow::MemoryPool* pool); + + /// Reshape `array` to `read_type`, null-filling nested fields added by schema + /// evolution. No-op when types match. STRUCT matches children by paimon field id; + /// LIST/MAP recurse into items, preserving offsets and validity. + static Result> AlignArrayToReadType( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool); + + private: + static Result HasNestedSubfieldProjectionType( + const std::shared_ptr& file_type, + const std::shared_ptr& read_type); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp new file mode 100644 index 00000000..7905b2ce --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -0,0 +1,737 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/nested_projection_utils.h" + +#include "arrow/array/array_dict.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_dict.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/ipc/json_simple.h" +#include "arrow/memory_pool.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +// Helper: create an arrow::Field with paimon.id metadata +static std::shared_ptr MakeField(const std::string& name, + const std::shared_ptr& type, + int32_t paimon_id) { + DataField data_field(paimon_id, arrow::field(name, type)); + return DataField::ConvertDataFieldToArrowField(data_field); +} + +// Helper: a variant-access projection, i.e. a struct whose children carry `__VARIANT_METADATA` +// descriptions. +static std::shared_ptr MakeVariantAccessType() { + auto child = arrow::field( + "0", arrow::int64(), /*nullable=*/true, + arrow::KeyValueMetadata::Make( + {DataField::DESCRIPTION}, + {VariantAccessUtils::BuildVariantMetadata("$.x", /*fail_on_error=*/false, "UTC")})); + return arrow::struct_({child}); +} + +// ============== GetPaimonFieldId ============== + +TEST(NestedProjectionUtilsTest, GetPaimonFieldIdPresent) { + auto field = MakeField("col", arrow::int32(), 42); + ASSERT_OK_AND_ASSIGN(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field)); + ASSERT_EQ(field_id, 42); +} + +TEST(NestedProjectionUtilsTest, GetPaimonFieldIdMissing) { + auto field = arrow::field("col", arrow::int32()); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::GetPaimonFieldId(field), + "do not exist metadata in field"); +} + +// ============== FindFieldByPaimonId ============== + +TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdFound) { + auto struct_type = + arrow::struct_({MakeField("x", arrow::int32(), 1), MakeField("y", arrow::utf8(), 2)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr found, + NestedProjectionUtils::FindFieldByPaimonId(struct_type, 2)); + ASSERT_NE(found, nullptr); + ASSERT_EQ(found->name(), "y"); +} + +TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdNotFound) { + auto struct_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::FindFieldByPaimonId(struct_type, 99), + "cannot find field 99"); +} + +// ============== PruneDataType ============== + +TEST(NestedProjectionUtilsTest, PruneDataTypeIdenticalTypes) { + auto type = arrow::int32(); + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(type, type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(type)); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeAtomicType) { + // Different atomic types: return data_type + auto read_type = arrow::int64(); + auto data_type = arrow::int32(); + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(data_type)); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeStructPruneSubset) { + // data: STRUCT + // read: STRUCT + // expected: STRUCT + auto data_type = + arrow::struct_({MakeField("x", arrow::int32(), 1), MakeField("y", arrow::utf8(), 2), + MakeField("z", arrow::float64(), 3)}); + auto read_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(result.value()->num_fields(), 1); + ASSERT_EQ(result.value()->field(0)->name(), "x"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeStructAllFieldsPruned) { + // data: STRUCT + // read: STRUCT — no match + // expected: fail-fast (struct-internal schema evolution unsupported) + auto data_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + auto read_type = arrow::struct_({MakeField("y", arrow::int32(), 99)}); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "cannot find field 99 in struct type"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeNestedStruct) { + // data: STRUCT(id=1)> + // read: STRUCT(id=1)> + // expected: STRUCT(id=1)> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::struct_({MakeField("inner", inner_data, 1)}); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::struct_({MakeField("inner", inner_read, 1)}); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(result.value()->num_fields(), 1); + auto pruned_inner = result.value()->field(0)->type(); + ASSERT_EQ(pruned_inner->num_fields(), 1); + ASSERT_EQ(pruned_inner->field(0)->name(), "a"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithStructElement) { + // data: LIST> + // read: LIST> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::list(arrow::field("item", inner_data)); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::list(arrow::field("item", inner_read)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapWithStructValue) { + // data: MAP> + // read: MAP> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::map(arrow::utf8(), inner_data); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::map(arrow::utf8(), inner_read); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside map"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithVariantAccessElement) { + // data: LIST, read: LIST + // Not a projection of the list itself, so it must pass through to the variant read plans. + auto data_type = arrow::list(arrow::field("element", VariantTypeUtils::UnshreddedStructType())); + auto read_type = arrow::list(arrow::field("element", MakeVariantAccessType())); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapWithVariantAccessValue) { + auto data_type = arrow::map(arrow::utf8(), VariantTypeUtils::UnshreddedStructType()); + auto read_type = arrow::map(arrow::utf8(), MakeVariantAccessType()); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithVariantAccessInsideStruct) { + // The variant sits one struct level below the list, next to a plain sibling that is kept. + auto data_inner = arrow::struct_({MakeField("v", VariantTypeUtils::UnshreddedStructType(), 10), + MakeField("t", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_( + {MakeField("v", MakeVariantAccessType(), 10), MakeField("t", arrow::utf8(), 11)}); + auto data_type = arrow::list(arrow::field("element", data_inner)); + auto read_type = arrow::list(arrow::field("element", read_inner)); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListDroppingSiblingOfVariantStillFails) { + // Dropping the plain sibling is a real partial projection inside the list and must keep + // failing fast, variant access or not. + auto data_inner = arrow::struct_({MakeField("v", VariantTypeUtils::UnshreddedStructType(), 10), + MakeField("t", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_({MakeField("v", MakeVariantAccessType(), 10)}); + auto data_type = arrow::list(arrow::field("element", data_inner)); + auto read_type = arrow::list(arrow::field("element", read_inner)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListStructSchemaEvolutionAddedField) { + // Added field (id=12) inside the list's struct: return the file struct. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::list(arrow::field("item", data_inner)); + auto read_type = arrow::list(arrow::field("item", read_inner)); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapStructSchemaEvolutionAddedField) { + // Added field inside a MAP value. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::map(arrow::utf8(), data_inner); + auto read_type = arrow::map(arrow::utf8(), read_inner); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListStructDropAndAddStillFails) { + // Dropping a file field (b) is a real partial projection -- keep failing. + auto data_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto read_inner = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("c", arrow::int32(), 12)}); + auto data_type = arrow::list(arrow::field("item", data_inner)); + auto read_type = arrow::list(arrow::field("item", read_inner)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructField) { + auto* pool = arrow::default_memory_pool(); + // file array: list> = [ [{1,"x"},{2,"y"}], [{3,"z"}] ] + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1, 2, 3}).ok()); + std::shared_ptr a; + ASSERT_TRUE(ab.Finish(&a).ok()); + arrow::StringBuilder bb(pool); + ASSERT_TRUE(bb.AppendValues({"x", "y", "z"}).ok()); + std::shared_ptr b; + ASSERT_TRUE(bb.Finish(&b).ok()); + auto data_struct_type = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + std::shared_ptr struct_arr = + arrow::StructArray::Make({a, b}, data_struct_type->fields()).ValueOrDie(); + arrow::Int32Builder offb(pool); + ASSERT_TRUE(offb.AppendValues({0, 2, 3}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offb.Finish(&offsets).ok()); + std::shared_ptr list_arr = + arrow::ListArray::FromArrays(*offsets, *struct_arr, pool).ValueOrDie(); + + // read type adds c:int(12) inside the struct. + auto read_struct = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11), + MakeField("c", arrow::int32(), 12)}); + auto read_type = arrow::list(arrow::field("item", read_struct)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(list_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString(); + auto out_struct = std::static_pointer_cast( + std::static_pointer_cast(aligned)->values()); + auto c_col = out_struct->GetFieldByName("c"); + ASSERT_NE(c_col, nullptr); + ASSERT_EQ(c_col->null_count(), c_col->length()); // added field is all null + auto a_col = std::static_pointer_cast(out_struct->GetFieldByName("a")); + ASSERT_EQ(a_col->Value(0), 1); + ASSERT_EQ(a_col->Value(2), 3); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeDecodesDictionaryLeafAndNullFills) { + // ORC lazy decoding returns dictionary; decode/cast to string. + auto* pool = arrow::default_memory_pool(); + auto dict_type = arrow::dictionary(arrow::int64(), arrow::large_utf8()); + auto a_dict = + arrow::ipc::internal::json::ArrayFromJSON(dict_type, R"(["x", "y", "x"])").ValueOrDie(); + auto data_struct = arrow::struct_({MakeField("a", dict_type, 10)}); + auto struct_arr = arrow::StructArray::Make({a_dict}, data_struct->fields()).ValueOrDie(); + + auto read_type = + arrow::struct_({MakeField("a", arrow::utf8(), 10), MakeField("b", arrow::int32(), 11)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString(); + auto out = std::static_pointer_cast(aligned); + ASSERT_EQ(std::static_pointer_cast(out->GetFieldByName("a"))->GetString(0), + "x"); + auto b = out->GetFieldByName("b"); + ASSERT_EQ(b->null_count(), b->length()); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeAppliesReadNullability) { + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1, 2}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_field = DataField::ConvertDataFieldToArrowField( + DataField(10, arrow::field("a", arrow::int32(), /*nullable=*/false))); + auto struct_arr = arrow::StructArray::Make({a_arr}, {data_field}).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + ASSERT_TRUE(aligned->type()->field(0)->nullable()); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeFieldIdChangeNullFillsNotLeak) { + // a(id=10) replaced by a(id=11), same name/type: new field must read null, not leak. + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({42}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 11)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + auto a_out = std::static_pointer_cast(aligned)->GetFieldByName("a"); + ASSERT_NE(a_out, nullptr); + ASSERT_EQ(a_out->null_count(), a_out->length()); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeRejectsLeafTypeChange) { + auto* pool = arrow::default_memory_pool(); + arrow::Int32Builder ab(pool); + ASSERT_TRUE(ab.AppendValues({1}).ok()); + std::shared_ptr a_arr; + ASSERT_TRUE(ab.Finish(&a_arr).ok()); + auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_({MakeField("a", arrow::int64(), 10)}); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool), + "AlignArrayToReadType unsupported leaf type change"); +} + +TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsNestedLargeBinaryBlob) { + auto* pool = arrow::default_memory_pool(); + arrow::LargeBinaryBuilder blobb(pool); + ASSERT_TRUE(blobb.AppendValues({"a", "b"}).ok()); + std::shared_ptr blob; + ASSERT_TRUE(blobb.Finish(&blob).ok()); + auto data_struct = arrow::struct_({MakeField("blob", arrow::large_binary(), 10)}); + auto struct_arr = arrow::StructArray::Make({blob}, data_struct->fields()).ValueOrDie(); + auto read_type = arrow::struct_( + {MakeField("blob", arrow::large_binary(), 10), MakeField("c", arrow::int32(), 11)}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned, + NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool)); + auto blob_out = std::static_pointer_cast(aligned)->GetFieldByName("blob"); + ASSERT_EQ(blob_out->type_id(), arrow::Type::LARGE_BINARY); + ASSERT_TRUE(blob_out->Equals(*blob)); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_FALSE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionWithProjection) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField( + "f1", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("b", arrow::utf8(), 4)}), + 3), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_TRUE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionTypeMismatchReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::list(arrow::field("item", arrow::int32())), 1), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "requires same nested type kind"); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionAtomicTypeMismatchReturnsFalse) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::map(arrow::utf8(), arrow::int32()), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::map(arrow::utf8(), arrow::int16()), 1), + }); + + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_FALSE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionMissingStructChildReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField( + "f0", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("b", arrow::utf8(), 3)}), + 1), + }); + auto read_schema = arrow::schema({ + MakeField( + "f0", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("c", arrow::utf8(), 4)}), + 1), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "requested struct child"); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionMissingTopLevelFieldReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "missing in file schema"); +} + +// ============== GetMapSelectedKeys ============== +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysPresent) { + auto metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"key1,key2,key3"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(keys[0], "key1"); + ASSERT_EQ(keys[1], "key2"); + ASSERT_EQ(keys[2], "key3"); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysAbsent) { + auto field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_TRUE(keys.empty()); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysEmptyString) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {""}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 1); + ASSERT_EQ(keys[0], ""); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysContainsEmptyToken) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a, ,b"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(keys[0], "a"); + ASSERT_EQ(keys[1], " "); + ASSERT_EQ(keys[2], "b"); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysWhitespaceOnly) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {" "}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 1); + ASSERT_EQ(keys[0], " "); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysDuplicateKey) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,a"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::GetMapSelectedKeys(field), + "Duplicate selected key 'a'"); +} + +// ============== FilterMapArrayBySelectedKeys ============== + +class NestedProjectionUtilsMapArrayTest : public ::testing::Test { + protected: + // Helper to build a MapArray from vectors of key-value pairs. + static std::shared_ptr BuildStringInt32MapArray( + const std::vector>>& maps, + const std::vector& null_mask = {}) { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + for (size_t i = 0; i < maps.size(); ++i) { + if (!null_mask.empty() && !null_mask[i]) { + EXPECT_TRUE(map_builder.AppendNull().ok()); + continue; + } + EXPECT_TRUE(map_builder.Append().ok()); + for (const auto& [k, v] : maps[i]) { + EXPECT_TRUE(key_builder->Append(k).ok()); + EXPECT_TRUE(value_builder->Append(v).ok()); + } + } + std::shared_ptr result; + EXPECT_TRUE(map_builder.Finish(&result).ok()); + return result; + } +}; + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysBasic) { + // Map with 3 entries each, select only "a" and "c" + auto map_array = BuildStringInt32MapArray({ + {{"a", 1}, {"b", 2}, {"c", 3}}, + {{"a", 10}, {"d", 40}}, + }); + + std::vector selected = {"a", "c"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + + auto expected = BuildStringInt32MapArray({ + {{"a", 1}, {"c", 3}}, + {{"a", 10}}, + }); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptySelectedKeys) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}}}); + std::vector empty_keys; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, empty_keys, arrow::default_memory_pool())); + // Should return original array unchanged + ASSERT_EQ(filtered.get(), map_array.get()); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysAllKept) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"a", "b"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + ASSERT_TRUE(filtered->Equals(map_array)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysNoneKept) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"x", "y"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptyStringKeySelected) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"", 9}, {"b", 2}}}); + std::vector selected = {""}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"", 9}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysWithNull) { + // maps[0] = {"a":1}, maps[1] = null, maps[2] = {"b":2,"c":3} + auto map_array = + BuildStringInt32MapArray({{{"a", 1}}, {}, {{"b", 2}, {"c", 3}}}, {true, false, true}); + + std::vector selected = {"a", "c"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"a", 1}}, {}, {{"c", 3}}}, {true, false, true}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptyArray) { + auto map_array = BuildStringInt32MapArray({}); + std::vector selected = {"a"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysSelectedOrderWins) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}, {"c", 3}}}); + std::vector selected = {"c", "a"}; + + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"c", 3}, {"a", 1}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDuplicateSelectedKeys) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"a", "a"}; + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool()), + "Duplicate selected key 'a'"); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDictionaryStringKey) { + auto map_array = BuildStringInt32MapArray({ + {{"a", 1}, {"b", 2}, {"c", 3}}, + {{"c", 30}, {"a", 10}}, + }); + auto map = std::static_pointer_cast(map_array); + auto string_keys = std::static_pointer_cast(map->keys()); + + arrow::StringDictionaryBuilder dict_builder(arrow::default_memory_pool()); + for (int64_t i = 0; i < string_keys->length(); ++i) { + ASSERT_TRUE(dict_builder.Append(string_keys->GetView(i)).ok()); + } + auto dict_keys_result = dict_builder.Finish(); + ASSERT_TRUE(dict_keys_result.ok()); + auto dict_keys = dict_keys_result.ValueUnsafe(); + + auto dict_map_type = arrow::map(dict_keys->type(), map->items()->type()); + auto dict_map_array = std::make_shared( + dict_map_type, map->length(), map->value_offsets(), dict_keys, map->items(), + map->null_bitmap(), map->null_count(), map->offset()); + + std::vector selected = {"c", "a"}; + ASSERT_OK_AND_ASSIGN(auto filtered, + NestedProjectionUtils::FilterMapArrayBySelectedKeys( + dict_map_array, selected, arrow::default_memory_pool())); + + auto expected = BuildStringInt32MapArray({ + {{"c", 3}, {"a", 1}}, + {{"c", 30}, {"a", 10}}, + }); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDictionaryLargeStringKey) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + auto map = std::static_pointer_cast(map_array); + + arrow::LargeStringBuilder dict_value_builder(arrow::default_memory_pool()); + ASSERT_TRUE(dict_value_builder.Append("a").ok()); + ASSERT_TRUE(dict_value_builder.Append("b").ok()); + std::shared_ptr dict_values; + ASSERT_TRUE(dict_value_builder.Finish(&dict_values).ok()); + + arrow::Int64Builder index_builder(arrow::default_memory_pool()); + ASSERT_TRUE(index_builder.Append(0).ok()); + ASSERT_TRUE(index_builder.Append(1).ok()); + std::shared_ptr indices; + ASSERT_TRUE(index_builder.Finish(&indices).ok()); + + auto dict_type = arrow::dictionary(arrow::int64(), arrow::large_utf8()); + auto large_string_dict_keys_result = + arrow::DictionaryArray::FromArrays(dict_type, indices, dict_values); + ASSERT_TRUE(large_string_dict_keys_result.ok()); + auto large_string_dict_keys = large_string_dict_keys_result.ValueUnsafe(); + + auto dict_map_type = arrow::map(large_string_dict_keys->type(), map->items()->type()); + auto dict_map_array = std::make_shared( + dict_map_type, map->length(), map->value_offsets(), large_string_dict_keys, map->items(), + map->null_bitmap(), map->null_count(), map->offset()); + + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + dict_map_array, {"a"}, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"a", 1}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h index b62bdc8b..1e334704 100644 --- a/src/paimon/core/utils/objects_file.h +++ b/src/paimon/core/utils/objects_file.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include @@ -26,6 +27,8 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/cache/cache.h" #include "paimon/common/data/columnar/columnar_row.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -39,6 +42,8 @@ #include "paimon/format/reader_builder.h" #include "paimon/format/writer_builder.h" #include "paimon/fs/file_system.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/memory/bytes.h" #include "paimon/record_batch.h" namespace paimon { @@ -52,7 +57,7 @@ class ObjectsFile { const std::shared_ptr& writer_builder, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + const std::shared_ptr& cache, const std::shared_ptr& pool); virtual ~ObjectsFile() = default; @@ -82,6 +87,9 @@ class ObjectsFile { std::shared_ptr file_system_; std::shared_ptr reader_builder_; std::string compression_; + std::shared_ptr cache_; + + Result ReadFileSegment(const std::string& file_path) const; }; template @@ -91,6 +99,7 @@ ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, + const std::shared_ptr& cache, const std::shared_ptr& pool) : path_factory_(path_factory), pool_(pool), @@ -98,9 +107,8 @@ ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, writer_builder_(std::move(writer_builder)), file_system_(file_system), reader_builder_(std::move(reader_builder)), - compression_(compression) { - // TODO(xinyu.lxy): add cache -} + compression_(compression), + cache_(cache) {} template Status ObjectsFile::ReadIfFileExist(const std::string& file_name, @@ -119,8 +127,33 @@ Status ObjectsFile::Read(const std::string& file_name, const std::function(const T&)>& filter, std::vector* result) const { std::string file_path = path_factory_->ToPath(file_name); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_input_stream, - file_system_->Open(file_path)); + std::shared_ptr file_input_stream; + std::shared_ptr cached_bytes; + if (cache_) { + // Use a whole-file key so cache hits do not need a metadata lookup just to discover file + // length. + auto cache_key = + CacheKey::ForKind(file_path, /*position=*/0, /*length=*/-1, CacheKind::MANIFEST); + auto supplier = + [this, + &file_path](const std::shared_ptr&) -> Result> { + PAIMON_ASSIGN_OR_RAISE(MemorySegment segment, ReadFileSegment(file_path)); + return std::make_shared(segment, CacheCallback()); + }; + Result> cache_result = cache_->Get(cache_key, supplier); + if (cache_result.ok() && cache_result.value() && + cache_result.value()->GetSegment().Data() != nullptr) { + cached_bytes = cache_result.value()->GetSegment().GetOrCreateHeapMemory(pool_.get()); + file_input_stream = + std::make_shared(cached_bytes->data(), cached_bytes->size()); + } + } + if (!file_input_stream) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr unique_file_input_stream, + file_system_->Open(file_path)); + file_input_stream = std::shared_ptr(std::move(unique_file_input_stream)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, reader_builder_->Build(file_input_stream)); auto reader = std::make_unique(std::move(batch_reader), @@ -156,6 +189,24 @@ Status ObjectsFile::Read(const std::string& file_name, return Status::OK(); } +template +Result ObjectsFile::ReadFileSegment(const std::string& file_path) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, + file_system_->Open(file_path)); + PAIMON_ASSIGN_OR_RAISE(int64_t input_length, input_stream->Length()); + + PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); + auto bytes = std::make_shared(input_length, pool_.get()); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_size, + input_stream->Read(bytes->data(), input_length)); + if (actual_read_size != input_length) { + return Status::IOError(fmt::format( + "Unexpected EOF while reading manifest file {}, expected {} bytes, got {} bytes", + file_path, input_length, actual_read_size)); + } + return MemorySegment::Wrap(bytes); +} + template Result> ObjectsFile::WriteWithoutRolling( const std::vector& records) { diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp index 448a495f..a205f1e2 100644 --- a/src/paimon/core/utils/primary_key_table_utils_test.cpp +++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp @@ -20,15 +20,23 @@ #include #include +#include +#include #include #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/row_kind.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/core/core_options.h" +#include "paimon/core/key_value.h" +#include "paimon/core/mergetree/compact/merge_function.h" #include "paimon/defs.h" +#include "paimon/memory/memory_pool.h" #include "paimon/status.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/key_value_checker.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -56,4 +64,52 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateSequenceFieldsComparator) { } } +TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete) { + auto pool = GetDefaultPool(); + auto value_schema = arrow::schema({arrow::field("v0", arrow::int32())}); + + // ignore-delete can also be configured through the merge-engine-specific + // first-row.ignore-delete key, which must reach FirstRowMergeFunction the same way. + for (const char* ignore_delete_key : + {Options::IGNORE_DELETE, Options::FALLBACK_FIRST_ROW_IGNORE_DELETE}) { + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"}, + {ignore_delete_key, "true"}})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options)); + merge_function->Reset(); + + KeyValue insert_kv(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get())); + KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/1, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get())); + ASSERT_OK(merge_function->Add(std::move(insert_kv))); + ASSERT_OK(merge_function->Add(std::move(delete_kv))); + + ASSERT_OK_AND_ASSIGN(std::optional result_kv, merge_function->GetResult()); + ASSERT_TRUE(result_kv.has_value()); + KeyValue expected(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get())); + KeyValueChecker::CheckResult(expected, result_kv.value(), /*key_arity=*/1, + /*value_arity=*/1); + } + + // Without the option, the first-row merge engine still rejects retract records. + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"}})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr merge_function, + PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options)); + merge_function->Reset(); + KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/0, /*level=*/0, /*key=*/ + BinaryRowGenerator::GenerateRowPtr({10}, pool.get()), + /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get())); + ASSERT_NOK_WITH_MSG(merge_function->Add(std::move(delete_kv)), + "First row merge engine can not accept DELETE/UPDATE_BEFORE records"); +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/tag_manager.cpp b/src/paimon/core/utils/tag_manager.cpp index a7c28a04..25cdba2e 100644 --- a/src/paimon/core/utils/tag_manager.cpp +++ b/src/paimon/core/utils/tag_manager.cpp @@ -18,12 +18,14 @@ #include "paimon/core/utils/tag_manager.h" +#include #include -#include #include +#include #include "fmt/format.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/file_system.h" @@ -55,8 +57,35 @@ Result> TagManager::Get(const std::string& tag_name) const { return std::optional(std::move(tag)); } +Result> TagManager::ListTagNames() const { + std::vector tag_names; + std::string tag_dir = TagDirectory(); + PAIMON_ASSIGN_OR_RAISE(bool is_exist, fs_->Exists(tag_dir)); + if (!is_exist) { + return tag_names; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs_->ListDir(tag_dir, &file_status_list)); + std::string tag_prefix = TAG_PREFIX; + for (const auto& file_status : file_status_list) { + if (file_status->IsDir()) { + continue; + } + std::string file_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(file_name, tag_prefix, /*start_pos=*/0)) { + tag_names.push_back(file_name.substr(tag_prefix.length())); + } + } + std::sort(tag_names.begin(), tag_names.end()); + return tag_names; +} + std::string TagManager::TagPath(const std::string& tag_name) const { - return PathUtil::JoinPath(BranchManager::BranchPath(root_path_, branch_), - "/tag/" + std::string(TAG_PREFIX) + tag_name); + return PathUtil::JoinPath(TagDirectory(), std::string(TAG_PREFIX) + tag_name); +} + +std::string TagManager::TagDirectory() const { + return PathUtil::JoinPath(BranchManager::BranchPath(root_path_, branch_), "tag"); } } // namespace paimon diff --git a/src/paimon/core/utils/tag_manager.h b/src/paimon/core/utils/tag_manager.h index 27742418..d1b937e8 100644 --- a/src/paimon/core/utils/tag_manager.h +++ b/src/paimon/core/utils/tag_manager.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "paimon/core/tag/tag.h" @@ -41,8 +42,12 @@ class TagManager { Result> Get(const std::string& tag_name) const; + Result> ListTagNames() const; + std::string TagPath(const std::string& tag_name) const; + std::string TagDirectory() const; + private: std::shared_ptr fs_; std::string root_path_; diff --git a/src/paimon/core/utils/tag_manager_test.cpp b/src/paimon/core/utils/tag_manager_test.cpp index 8909b1a1..0a36f014 100644 --- a/src/paimon/core/utils/tag_manager_test.cpp +++ b/src/paimon/core/utils/tag_manager_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/utils/tag_manager.h" #include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/testing/utils/testharness.h" @@ -43,4 +44,46 @@ TEST(TagManagerTest, TestTagPath) { TagManager(nullptr, "/root", "data").TagPath("data")); } +TEST(TagManagerTest, TestListTagNames) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string tag_dir = PathUtil::JoinPath(table_path, "tag"); + ASSERT_OK(fs->Mkdirs(tag_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-2"), "", /*overwrite=*/true)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-1"), "", /*overwrite=*/true)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "snapshot-1"), "", /*overwrite=*/true)); + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(tag_dir, "tag-dir"))); + + TagManager manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_EQ(tag_names, (std::vector{"1", "2"})); +} + +TEST(TagManagerTest, TestListBranchTagNames) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string tag_dir = PathUtil::JoinPath(BranchManager::BranchPath(table_path, "dev"), "tag"); + ASSERT_OK(fs->Mkdirs(tag_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-dev"), "", /*overwrite=*/true)); + + TagManager manager(fs, table_path, "dev"); + ASSERT_EQ(manager.TagDirectory(), tag_dir); + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_EQ(tag_names, (std::vector{"dev"})); +} + +TEST(TagManagerTest, TestListTagNamesReturnsEmptyForMissingDirectory) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + TagManager manager(fs, PathUtil::JoinPath(dir->Str(), "table")); + + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_TRUE(tag_names.empty()); +} + } // namespace paimon::test diff --git a/src/paimon/format/avro/CMakeLists.txt b/src/paimon/format/avro/CMakeLists.txt index 8aa593b0..1c425674 100644 --- a/src/paimon/format/avro/CMakeLists.txt +++ b/src/paimon/format/avro/CMakeLists.txt @@ -40,7 +40,6 @@ if(PAIMON_ENABLE_AVRO) fmt avro tbb - dl Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -61,10 +60,8 @@ if(PAIMON_ENABLE_AVRO) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_avro_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN} EXTRA_LINK_LIBS avro) diff --git a/src/paimon/format/avro/avro_direct_decoder.cpp b/src/paimon/format/avro/avro_direct_decoder.cpp index 34d8d7db..dbc1252a 100644 --- a/src/paimon/format/avro/avro_direct_decoder.cpp +++ b/src/paimon/format/avro/avro_direct_decoder.cpp @@ -289,8 +289,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, case arrow::Type::DATE32: { if (logical_type.type() != ::avro::LogicalType::Type::DATE) { return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - arrow_type->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), arrow_type->ToString())); } auto* builder = arrow::internal::checked_cast(array_builder); @@ -299,8 +299,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - arrow_type->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), arrow_type->ToString())); } } @@ -332,8 +332,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - array_builder->type()->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), array_builder->type()->ToString())); } } @@ -379,8 +379,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - array_builder->type()->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), array_builder->type()->ToString())); } } @@ -398,7 +398,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, return DecodeMapToBuilder(avro_node, decoder, array_builder, ctx); } default: - return Status::Invalid(fmt::format("Unsupported avro type: {}", type)); + return Status::Invalid( + fmt::format("Unsupported avro type: {}", ::avro::toString(type))); } } diff --git a/src/paimon/format/avro/avro_direct_decoder.h b/src/paimon/format/avro/avro_direct_decoder.h index 41e54f45..c507091a 100644 --- a/src/paimon/format/avro/avro_direct_decoder.h +++ b/src/paimon/format/avro/avro_direct_decoder.h @@ -33,6 +33,9 @@ namespace paimon::avro { class AvroDirectDecoder { public: + AvroDirectDecoder() = delete; + ~AvroDirectDecoder() = delete; + /// Context for reusing scratch buffers during Avro decoding /// /// Avoids frequent small allocations by reusing temporary buffers across multiple decode diff --git a/src/paimon/format/avro/avro_direct_encoder.cpp b/src/paimon/format/avro/avro_direct_encoder.cpp index 8a4d1b92..b371c27f 100644 --- a/src/paimon/format/avro/avro_direct_encoder.cpp +++ b/src/paimon/format/avro/avro_direct_encoder.cpp @@ -224,7 +224,14 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, return Status::OK(); } - // Handle regular BYTES + // Handle regular BYTES (binary or large_binary) + if (array.type()->id() == arrow::Type::LARGE_BINARY) { + const auto& large_binary_array = + arrow::internal::checked_cast(array); + std::string_view value = large_binary_array.GetView(row_index); + encoder->encodeBytes(reinterpret_cast(value.data()), value.size()); + return Status::OK(); + } const auto& binary_array = arrow::internal::checked_cast(array); std::string_view value = binary_array.GetView(row_index); @@ -347,11 +354,11 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, return Status::Invalid(fmt::format("AVRO_MAP keys must be StringArray, got {}", keys->type()->ToString())); } + const auto& string_array = + arrow::internal::checked_cast(*keys); for (int64_t i = start; i < end; ++i) { encoder->startItem(); - const auto& string_array = - arrow::internal::checked_cast(*keys); std::string_view key_value = string_array.GetView(i); encoder->encodeString(std::string(key_value)); diff --git a/src/paimon/format/avro/avro_direct_encoder.h b/src/paimon/format/avro/avro_direct_encoder.h index 4b5a998d..646e19e5 100644 --- a/src/paimon/format/avro/avro_direct_encoder.h +++ b/src/paimon/format/avro/avro_direct_encoder.h @@ -33,6 +33,9 @@ namespace paimon::avro { class AvroDirectEncoder { public: + AvroDirectEncoder() = delete; + ~AvroDirectEncoder() = delete; + /// Context for reusing scratch buffers during Avro encoding /// /// Avoids frequent small allocations by reusing temporary buffers across multiple encode diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index d2d2f538..92ac769c 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include "paimon/format/avro/avro_file_batch_reader.h" +#include #include #include @@ -28,6 +29,7 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/avro/avro_input_stream_impl.h" #include "paimon/format/avro/avro_schema_converter.h" #include "paimon/reader/batch_reader.h" @@ -116,6 +118,7 @@ Result AvroFileBatchReader::NextBatch() { previous_first_row_ = next_row_to_read_; next_row_to_read_ += array_builder_->length(); if (array_builder_->length() == 0) { + previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, @@ -123,6 +126,7 @@ Result AvroFileBatchReader::NextBatch() { std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); + previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); } catch (const ::avro::Exception& e) { return Status::Invalid(fmt::format("avro reader next batch failed. {}", e.what())); @@ -143,18 +147,34 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, if (selection_bitmap) { // TODO(menglingda.mld): support bitmap } - previous_first_row_ = std::numeric_limits::max(); - next_row_to_read_ = std::numeric_limits::max(); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_read_schema, arrow::ImportSchema(read_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, ArrowUtils::DataTypeToSchema(file_data_type_)); + PAIMON_ASSIGN_OR_RAISE( + bool has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, arrow_read_schema)); + if (has_nested_projection) { + return Status::Invalid( + "SetReadSchema failed: avro reader does not support nested sub-field projection"); + } PAIMON_ASSIGN_OR_RAISE(read_fields_projection_, CalculateReadFieldsProjection(file_schema, arrow_read_schema->fields())); - array_builder_->Reset(); std::shared_ptr<::arrow::DataType> read_data_type = arrow::struct_(arrow_read_schema->fields()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array_builder_, + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr array_builder, arrow::MakeBuilder(read_data_type, arrow_pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::avro::DataFileReaderBase> reader, + CreateDataFileReader(input_stream_, pool_)); + + if (reader_) { + reader_->close(); + } + reader_ = std::move(reader); + array_builder_ = std::move(array_builder); + previous_first_row_ = std::numeric_limits::max(); + previous_batch_row_count_ = 0; + next_row_to_read_ = std::numeric_limits::max(); + close_ = false; return Status::OK(); } diff --git a/src/paimon/format/avro/avro_file_batch_reader.h b/src/paimon/format/avro/avro_file_batch_reader.h index e54a936e..ccd1f770 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.h +++ b/src/paimon/format/avro/avro_file_batch_reader.h @@ -18,12 +18,14 @@ #pragma once +#include #include #include #include #include #include "avro/DataFile.hh" +#include "fmt/format.h" #include "paimon/format/avro/avro_direct_decoder.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" @@ -47,8 +49,20 @@ class AvroFileBatchReader : public FileBatchReader { Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) override; - Result GetPreviousBatchFirstRowNumber() const override { - return previous_first_row_; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (previous_batch_row_count_ == 0) { + if (previous_first_row_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); + } + return previous_first_row_ + batch_row_id; } Result GetNumberOfRows() const override; @@ -92,6 +106,7 @@ class AvroFileBatchReader : public FileBatchReader { std::optional> read_fields_projection_; uint64_t previous_first_row_ = std::numeric_limits::max(); uint64_t next_row_to_read_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; mutable std::optional total_rows_ = std::nullopt; const int32_t batch_size_; bool close_ = false; diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index fa506796..0a0a858d 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -297,7 +297,39 @@ TEST_F(AvroFileBatchReaderTest, TestReadMapTypes) { ASSERT_TRUE(expected_array->Equals(result_array)); } -TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { +TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaRejectNestedSubFieldProjection) { + std::string path = PathUtil::JoinPath(dir_->Str(), "nested_projection_unsupported.avro"); + + arrow::FieldVector write_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8())}))}; + auto write_type = arrow::struct_(write_fields); + auto write_array = arrow::ipc::internal::json::ArrayFromJSON(write_type, R"([ + [1, [10, "x"]], + [2, [20, "y"]] + ])") + .ValueOrDie(); + WriteData(write_array, path, /*compression=*/"null"); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format_->CreateReaderBuilder(/*batch_size=*/1024)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(path)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(in)); + + arrow::FieldVector read_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32())}))}; + auto read_schema = arrow::schema(read_fields); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "does not support nested sub-field projection"); +} + +TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFileRowId) { std::string path = paimon::test::GetDataDir() + "/avro/append_simple.db/" "append_simple/bucket-0/" @@ -322,29 +354,76 @@ TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { ASSERT_OK_AND_ASSIGN(auto num_rows, reader->GetNumberOfRows()); ASSERT_EQ(4, num_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); ArrowArrayRelease(batch1.first.get()); ArrowSchemaRelease(batch1.second.get()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); - ASSERT_EQ(1, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(1, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch2.first.get()); ArrowSchemaRelease(batch2.second.get()); ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); - ASSERT_EQ(2, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch3.first.get()); ArrowSchemaRelease(batch3.second.get()); ASSERT_OK_AND_ASSIGN(auto batch4, reader->NextBatch()); - ASSERT_EQ(3, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(3, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch4.first.get()); ArrowSchemaRelease(batch4.second.get()); ASSERT_OK_AND_ASSIGN(auto batch5, reader->NextBatch()); - ASSERT_EQ(4, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch5)); } +TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaResetsReaderToFirstRow) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); + + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + }; + auto file_data_type = arrow::struct_(fields); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(file_data_type, R"([ + [1, 10], + [2, 20], + [3, 30], + [4, 40] + ])") + .ValueOrDie(); + WriteData(src_array, file_path, /*compression=*/"null"); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format_->CreateReaderBuilder(/*batch_size=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(in)); + + ASSERT_OK_AND_ASSIGN(auto first_batch, reader->NextBatch()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); + auto first_array = + arrow::ImportArray(first_batch.first.get(), first_batch.second.get()).ValueOrDie(); + ASSERT_TRUE(first_array->Equals(src_array->Slice(0, 2))) << first_array->ToString(); + + auto read_schema = arrow::schema({arrow::field("f1", arrow::int32())}); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + + ASSERT_OK_AND_ASSIGN(auto projected_batch, reader->NextBatch()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); + auto projected_array = + arrow::ImportArray(projected_batch.first.get(), projected_batch.second.get()).ValueOrDie(); + auto expected_projected_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("f1", arrow::int32())}), + R"([ + [10], + [20] + ])") + .ValueOrDie(); + ASSERT_TRUE(projected_array->Equals(expected_projected_array)) << projected_array->ToString(); +} + TEST_F(AvroFileBatchReaderTest, TestGetNumberOfRows) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); @@ -405,6 +484,56 @@ TEST_F(AvroFileBatchReaderTest, TestGetNumberOfRows) { } } +TEST_F(AvroFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_data_type = arrow::struct_({write_field}); + auto write_array = + arrow::ipc::internal::json::ArrayFromJSON(write_data_type, data_json).ValueOrDie(); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), file_name); + WriteData(write_array, file_path, /*compression=*/"null"); + + // Read back with binary schema + auto read_field = arrow::field("f0", arrow::binary()); + auto read_data_type = arrow::struct_({read_field}); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format_->CreateReaderBuilder(/*batch_size=*/1024)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(in)); + + // Check GetFileSchema: regardless of write type, avro file schema is always binary + ASSERT_OK_AND_ASSIGN(auto c_file_schema, batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + arrow::Schema expected_file_schema({read_field}); + ASSERT_TRUE(file_schema->Equals(expected_file_schema)); + + auto read_schema = arrow::schema({read_field}); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + EXPECT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto result_array, ::paimon::test::ReadResultCollector::CollectResult( + batch_reader.get())); + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(read_data_type, data_json).ValueOrDie(); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.avro"); + check_binary_read_result(arrow::large_binary(), "large-binary.avro"); +} + INSTANTIATE_TEST_SUITE_P(TestParam, AvroFileBatchReaderTest, ::testing::Values(false, true)); } // namespace paimon::avro::test diff --git a/src/paimon/format/avro/avro_file_format.cpp b/src/paimon/format/avro/avro_file_format.cpp index 1cbef060..e69b8e03 100644 --- a/src/paimon/format/avro/avro_file_format.cpp +++ b/src/paimon/format/avro/avro_file_format.cpp @@ -41,7 +41,7 @@ AvroFileFormat::AvroFileFormat(const std::map& options Result> AvroFileFormat::CreateReaderBuilder( int32_t batch_size) const { - return std::make_unique(options_, batch_size); + return std::make_unique(batch_size); } Result> AvroFileFormat::CreateWriterBuilder( diff --git a/src/paimon/format/avro/avro_format_writer.cpp b/src/paimon/format/avro/avro_format_writer.cpp index e4ad454a..21905644 100644 --- a/src/paimon/format/avro/avro_format_writer.cpp +++ b/src/paimon/format/avro/avro_format_writer.cpp @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include #include "arrow/api.h" @@ -111,6 +113,10 @@ Result AvroFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return false; } +Status AvroFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by avro format writer."); +} + Status AvroFormatWriter::AddBatch(ArrowArray* batch) { assert(batch); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, diff --git a/src/paimon/format/avro/avro_format_writer.h b/src/paimon/format/avro/avro_format_writer.h index 64427c31..23baffa9 100644 --- a/src/paimon/format/avro/avro_format_writer.h +++ b/src/paimon/format/avro/avro_format_writer.h @@ -20,8 +20,10 @@ #include #include +#include #include #include +#include #include "arrow/api.h" #include "avro/DataFile.hh" @@ -66,6 +68,8 @@ class AvroFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: static constexpr size_t DEFAULT_SYNC_INTERVAL = 64 * 1024; diff --git a/src/paimon/format/avro/avro_input_stream_impl.cpp b/src/paimon/format/avro/avro_input_stream_impl.cpp index b1d6a01c..d2dfddc6 100644 --- a/src/paimon/format/avro/avro_input_stream_impl.cpp +++ b/src/paimon/format/avro/avro_input_stream_impl.cpp @@ -27,6 +27,7 @@ #include #include "avro/Exception.hh" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -36,9 +37,10 @@ namespace paimon::avro { Result> AvroInputStreamImpl::Create( const std::shared_ptr& input_stream, size_t buffer_size, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "file length")); return std::unique_ptr( - new AvroInputStreamImpl(input_stream, buffer_size, length, pool)); + new AvroInputStreamImpl(input_stream, buffer_size, static_cast(length), pool)); } AvroInputStreamImpl::AvroInputStreamImpl(const std::shared_ptr& input_stream, @@ -69,9 +71,12 @@ bool AvroInputStreamImpl::next(const uint8_t** data, size_t* len) { if (remaining == 0) { return false; // eof } - auto read_length = - in_->Read(reinterpret_cast(buffer_), - static_cast(std::min(buffer_size_, remaining))); + uint64_t read_size = std::min(buffer_size_, remaining); + Status status = ValidateValueInRange(read_size, "read length"); + if (!status.ok()) { + throw ::avro::Exception("Read failed: {}", status.ToString()); + } + auto read_length = in_->Read(reinterpret_cast(buffer_), static_cast(read_size)); if (!read_length.ok()) { throw ::avro::Exception("Read failed: {}", read_length.status().ToString()); } diff --git a/src/paimon/format/avro/avro_output_stream_impl.cpp b/src/paimon/format/avro/avro_output_stream_impl.cpp index d70d2f97..6e3ce32e 100644 --- a/src/paimon/format/avro/avro_output_stream_impl.cpp +++ b/src/paimon/format/avro/avro_output_stream_impl.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -64,11 +65,16 @@ void AvroOutputStreamImpl::backup(size_t len) { void AvroOutputStreamImpl::FlushBuffer() { size_t length = buffer_size_ - available_; - Result write_len = out_->Write(reinterpret_cast(buffer_), length); + Status validate_status = ValidateValueInRange(length, "write length"); + if (!validate_status.ok()) { + throw std::runtime_error("write failed, status: " + validate_status.ToString()); + } + Result write_len = + out_->Write(reinterpret_cast(buffer_), static_cast(length)); if (!write_len.ok()) { throw std::runtime_error("write failed, status: " + write_len.status().ToString()); } - if (static_cast(write_len.value()) != length) { + if (write_len.value() != static_cast(length)) { throw std::runtime_error( fmt::format("write failed, expected length: {}, actual write length: {}", length, write_len.value())); diff --git a/src/paimon/format/avro/avro_reader_builder.h b/src/paimon/format/avro/avro_reader_builder.h index bd1da1a2..f19d8683 100644 --- a/src/paimon/format/avro/avro_reader_builder.h +++ b/src/paimon/format/avro/avro_reader_builder.h @@ -18,10 +18,7 @@ #pragma once -#include #include -#include -#include #include "avro/DataFile.hh" #include "paimon/format/avro/avro_file_batch_reader.h" @@ -34,8 +31,8 @@ namespace paimon::avro { class AvroReaderBuilder : public ReaderBuilder { public: - AvroReaderBuilder(const std::map& options, int32_t batch_size) - : batch_size_(batch_size), pool_(GetDefaultPool()), options_(options) {} + explicit AvroReaderBuilder(int32_t batch_size) + : batch_size_(batch_size), pool_(GetDefaultPool()) {} ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) override { pool_ = pool; @@ -50,7 +47,6 @@ class AvroReaderBuilder : public ReaderBuilder { private: const int32_t batch_size_; std::shared_ptr pool_; - const std::map options_; }; } // namespace paimon::avro diff --git a/src/paimon/format/avro/avro_schema_converter.cpp b/src/paimon/format/avro/avro_schema_converter.cpp index ef905fff..bfdbe5e0 100644 --- a/src/paimon/format/avro/avro_schema_converter.cpp +++ b/src/paimon/format/avro/avro_schema_converter.cpp @@ -31,6 +31,7 @@ #include "avro/Types.hh" #include "avro/ValidSchema.hh" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/format/avro/avro_file_format_factory.h" #include "paimon/format/avro/avro_utils.h" @@ -269,6 +270,7 @@ Result<::avro::Schema> AvroSchemaConverter::ArrowTypeToAvroSchema( case arrow::Type::STRING: return nullable ? NullableSchema(::avro::StringSchema()) : ::avro::StringSchema(); case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: return nullable ? NullableSchema(::avro::BytesSchema()) : ::avro::BytesSchema(); case arrow::Type::type::DATE32: { ::avro::IntSchema date_schema; @@ -379,6 +381,9 @@ Result<::avro::Schema> AvroSchemaConverter::ArrowTypeToAvroSchema( Result<::avro::ValidSchema> AvroSchemaConverter::ArrowSchemaToAvroSchema( const std::shared_ptr& arrow_schema) { + if (VariantTypeUtils::ContainsVariantField(arrow_schema)) { + return Status::NotImplemented("Avro format does not support the VARIANT type"); + } // top level row name of avro record, the same as java paimon static const std::string kTopLevelRowName = "org.apache.paimon.avro.generated.record"; ::avro::RecordSchema record_schema(kTopLevelRowName); diff --git a/src/paimon/format/avro/avro_schema_converter_test.cpp b/src/paimon/format/avro/avro_schema_converter_test.cpp index d61e930e..e30de486 100644 --- a/src/paimon/format/avro/avro_schema_converter_test.cpp +++ b/src/paimon/format/avro/avro_schema_converter_test.cpp @@ -22,6 +22,7 @@ #include "avro/Compiler.hh" #include "avro/ValidSchema.hh" #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/utils/versioned_object_serializer.h" @@ -246,4 +247,11 @@ TEST(AvroSchemaConverterTest, TestAvroSchemaToArrowDataTypeWithTimestampType) { ASSERT_TRUE(arrow_type->Equals(arrow::struct_(expected_fields))) << arrow_type->ToString(); } +TEST(AvroSchemaConverterTest, TestVariantNotSupported) { + auto arrow_schema = + arrow::schema({arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}); + auto result = AvroSchemaConverter::ArrowSchemaToAvroSchema(arrow_schema); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); +} + } // namespace paimon::avro::test diff --git a/src/paimon/format/avro/avro_stats_extractor.cpp b/src/paimon/format/avro/avro_stats_extractor.cpp index 104983d0..3bfbcf95 100644 --- a/src/paimon/format/avro/avro_stats_extractor.cpp +++ b/src/paimon/format/avro/avro_stats_extractor.cpp @@ -94,6 +94,7 @@ Result> AvroStatsExtractor::FetchColumnStatistics( case arrow::Type::type::DOUBLE: return ColumnStats::CreateDoubleColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::type::BINARY: + case arrow::Type::type::LARGE_BINARY: return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::type::STRING: return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); diff --git a/src/paimon/format/blob/CMakeLists.txt b/src/paimon/format/blob/CMakeLists.txt index 4400ba04..239a936a 100644 --- a/src/paimon/format/blob/CMakeLists.txt +++ b/src/paimon/format/blob/CMakeLists.txt @@ -27,7 +27,6 @@ add_paimon_lib(paimon_blob_file_format STATIC_LINK_LIBS arrow fmt - dl Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -47,9 +46,7 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_blob_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index 52df7979..3cd94ed0 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -19,7 +19,7 @@ #include "paimon/format/blob/blob_file_batch_reader.h" #include -#include +#include #include #include "arrow/api.h" @@ -29,7 +29,6 @@ #include "arrow/util/bit_util.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" -#include "paimon/common/executor/future.h" #include "paimon/common/io/offset_input_stream.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -42,7 +41,7 @@ namespace paimon::blob { Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, - const std::shared_ptr& pool) { + bool emit_placeholder_sentinel, const std::shared_ptr& pool) { if (input_stream == nullptr) { return Status::Invalid("blob file batch reader create failed: input stream is nullptr"); } @@ -52,26 +51,26 @@ Result> BlobFileBatchReader::Create( batch_size)); } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_size, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, input_stream->Length()); PAIMON_RETURN_NOT_OK( - input_stream->Seek(file_size - BlobDefs::kBlobFileHeaderLength, FS_SEEK_SET)); - int8_t header[BlobDefs::kBlobFileHeaderLength]; + input_stream->Seek(file_size - BlobDefs::kBlobFileFooterLength, FS_SEEK_SET)); + int8_t footer[BlobDefs::kBlobFileFooterLength]; PAIMON_ASSIGN_OR_RAISE( - int32_t actual_size, - input_stream->Read(reinterpret_cast(header), BlobDefs::kBlobFileHeaderLength)); - if (actual_size != BlobDefs::kBlobFileHeaderLength) { + int64_t actual_size, + input_stream->Read(reinterpret_cast(footer), BlobDefs::kBlobFileFooterLength)); + if (actual_size != BlobDefs::kBlobFileFooterLength) { return Status::Invalid( - fmt::format("actual read size {} not match with expect header length {}", actual_size, - BlobDefs::kBlobFileHeaderLength)); + fmt::format("actual read size {} not match with expect footer length {}", actual_size, + BlobDefs::kBlobFileFooterLength)); } - int8_t version = header[4]; + int8_t version = footer[4]; if (version != BlobDefs::kFileVersion) { return Status::Invalid(fmt::format( "create blob format reader failed. unsupported blob file version: {}", version)); } - int32_t index_length = GetIndexLength(header, 0); + int32_t index_length = GetIndexLength(footer, 0); PAIMON_RETURN_NOT_OK(input_stream->Seek( - file_size - BlobDefs::kBlobFileHeaderLength - index_length, FS_SEEK_SET)); + file_size - BlobDefs::kBlobFileFooterLength - index_length, FS_SEEK_SET)); std::vector index_bytes(index_length, '\0'); PAIMON_ASSIGN_OR_RAISE(actual_size, input_stream->Read(index_bytes.data(), index_length)); if (actual_size != index_length) { @@ -87,14 +86,15 @@ Result> BlobFileBatchReader::Create( int64_t offset = 0; for (const auto& blob_length : blob_lengths) { blob_offsets.push_back(offset); - // Null blobs (bin_length == -1) don't occupy file space + // null (-1) and placeholder (-2) entries occupy no file space if (blob_length >= 0) { offset += blob_length; } } PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri()); - auto reader = std::unique_ptr(new BlobFileBatchReader( - input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool)); + auto reader = std::unique_ptr( + new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size, + blob_as_descriptor, emit_placeholder_sentinel, pool)); return reader; } @@ -103,6 +103,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, bool blob_as_descriptor, + bool emit_placeholder_sentinel, const std::shared_ptr& pool) : input_stream_(input_stream), file_path_(file_path), @@ -112,6 +113,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp target_blob_offsets_(blob_offsets), batch_size_(batch_size), blob_as_descriptor_(blob_as_descriptor), + emit_placeholder_sentinel_(emit_placeholder_sentinel), pool_(pool), arrow_pool_(GetArrowPool(pool_)), metrics_(std::make_shared()) { @@ -160,8 +162,8 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, } target_type_ = arrow::struct_(arrow_schema->fields()); current_pos_ = 0; - previous_batch_first_row_number_ = std::numeric_limits::max(); - + previous_batch_start_pos_ = std::numeric_limits::max(); + previous_batch_row_count_ = 0; return Status::OK(); } @@ -172,11 +174,7 @@ Result> BlobFileBatchReader::NextBlobOffsets( PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0)); int64_t data_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - // Null blobs contribute zero bytes to content - if (!IsTargetNull(i)) { - data_length += GetTargetContentLength(i); - } + data_length += GetTargetOutputLength(current_pos_ + k); PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer, @@ -188,10 +186,7 @@ Result> BlobFileBatchReader::NextBlobContents( int32_t rows_to_read) const { int64_t total_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - if (!IsTargetNull(i)) { - total_length += GetTargetContentLength(i); - } + total_length += GetTargetOutputLength(current_pos_ + k); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer, arrow::AllocateBuffer(total_length, arrow_pool_.get())); @@ -201,6 +196,13 @@ Result> BlobFileBatchReader::NextBlobContents( if (IsTargetNull(i)) { continue; } + if (IsTargetPlaceholder(i)) { + // a placeholder entry has no data bytes in the file; emit the sentinel for the + // data-evolution blob fallback merge to identify it + memcpy(buffer, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength); + buffer += BlobDefs::kPlaceholderSentinelLength; + continue; + } int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); PAIMON_RETURN_NOT_OK(ReadBlobContentAt(offset, length, buffer)); @@ -272,6 +274,9 @@ Result> BlobFileBatchReader::BuildTargetArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); if (IsTargetNull(i)) { PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + } else if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); } else { int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); @@ -295,18 +300,31 @@ Result BlobFileBatchReader::NextBatch() { return Status::Invalid("target type is nullptr, call SetReadSchema first"); } if (current_pos_ >= target_blob_lengths_.size()) { - PAIMON_ASSIGN_OR_RAISE(previous_batch_first_row_number_, GetNumberOfRows()); + previous_batch_start_pos_ = target_blob_lengths_.size(); + previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } int32_t left_rows = target_blob_lengths_.size() - current_pos_; int32_t rows_to_read = std::min(left_rows, batch_size_); + if (!emit_placeholder_sentinel_) { + for (int32_t k = 0; k < rows_to_read; ++k) { + if (IsTargetPlaceholder(current_pos_ + k)) { + return Status::Invalid(fmt::format( + "blob file {} contains a placeholder entry (bin_length {}) written by a " + "data-evolution partial update; it can only be resolved by the data-evolution " + "blob fallback read path", + file_path_, BlobDefs::kPlaceholderBinLength)); + } + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array, BuildTargetArray(rows_to_read)); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get(), c_schema.get())); - previous_batch_first_row_number_ = target_blob_row_indexes_[current_pos_]; + previous_batch_start_pos_ = current_pos_; current_pos_ += rows_to_read; + previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); } diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 159d0552..ccb0afe8 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -26,6 +26,7 @@ #include "arrow/memory_pool.h" #include "arrow/type.h" +#include "fmt/format.h" #include "paimon/common/data/blob_defs.h" #include "paimon/fs/file_system.h" #include "paimon/memory/bytes.h" @@ -50,7 +51,8 @@ namespace paimon::blob { /// ==================================================================== /// 1. Data Bins Section /// ==================================================================== -/// The file consists of one or more contiguous 'bins' (bin_0, bin_1, bin_2, ...). +/// The file consists of zero or more contiguous 'bins' (bin_0, bin_1, bin_2, ...); rows whose +/// index entry is negative (see Section 2) have no bin. /// The structure of each bin is as follows: /// /// | Field Name | Length (bytes) | Description | @@ -68,7 +70,13 @@ namespace paimon::blob { /// ==================================================================== /// The Index is located after all data bins and is used for quick lookup and management. /// -/// Purpose: Records the lengths (record lens) of all data bins. +/// Purpose: Records one signed length entry per row, in row order. +/// +/// Special lengths: an entry does not always describe a bin present in the Data Bins Section. +/// - -1 (BlobDefs::kNullBinLength): a null blob; no bin is written. +/// - -2 (BlobDefs::kPlaceholderBinLength): a placeholder blob written by a data-evolution +/// partial update for a row it did not touch; no bin is written and the value must be +/// resolved from an older blob file covering the same row. /// /// Encoding: /// - Uses Delta Encoding to store differences between successive length values. @@ -88,9 +96,16 @@ namespace paimon::blob { /// - Current version is 1. class BlobFileBatchReader : public FileBatchReader { public: + /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == + /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving + /// them requires the data-evolution blob fallback path; when true they are returned as the + /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away. Stored values + /// are returned verbatim; see BlobDefs::kPlaceholderSentinel for the accepted collision + /// with a user value exactly equal to the sentinel. static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -99,14 +114,22 @@ class BlobFileBatchReader : public FileBatchReader { Result NextBatch() override; - Result GetPreviousBatchFirstRowNumber() const override { - if (all_blob_lengths_.size() != target_blob_lengths_.size()) { + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (previous_batch_row_count_ == 0) { + if (previous_batch_start_pos_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { return Status::Invalid( - "Cannot call GetPreviousBatchFirstRowNumber in BlobFileBatchReader because, after " - "bitmap pushdown, rows in the array returned by NextBatch are no longer " - "contiguous."); + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); } - return previous_batch_first_row_number_; + // target_blob_row_indexes_ maps every selected position back to its original file row + // index, so this stays correct after a bitmap selection removed rows. + return target_blob_row_indexes_[previous_batch_start_pos_ + batch_row_id]; } Result GetNumberOfRows() const override { @@ -133,7 +156,8 @@ class BlobFileBatchReader : public FileBatchReader { BlobFileBatchReader(const std::shared_ptr& input_stream, const std::string& file_path, const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const; @@ -149,6 +173,23 @@ class BlobFileBatchReader : public FileBatchReader { return target_blob_lengths_[index] == BlobDefs::kNullBinLength; } + bool IsTargetPlaceholder(size_t index) const { + return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength; + } + + /// Content bytes the blob at the given index contributes to the output buffer: nothing for + /// a null entry, the sentinel bytes for a placeholder entry (which occupies no file space), + /// the stored bytes otherwise. + int64_t GetTargetOutputLength(size_t index) const { + if (IsTargetNull(index)) { + return 0; + } + if (IsTargetPlaceholder(index)) { + return BlobDefs::kPlaceholderSentinelLength; + } + return GetTargetContentLength(index); + } + int64_t GetTargetContentOffset(size_t index) const { return target_blob_offsets_[index] + BlobDefs::kContentStartOffset; } @@ -168,6 +209,7 @@ class BlobFileBatchReader : public FileBatchReader { const int32_t batch_size_; const bool blob_as_descriptor_; + const bool emit_placeholder_sentinel_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; @@ -175,7 +217,10 @@ class BlobFileBatchReader : public FileBatchReader { std::shared_ptr metrics_; size_t current_pos_ = 0; - uint64_t previous_batch_first_row_number_ = std::numeric_limits::max(); + /// Start position of the previous batch in the (possibly selection-filtered) target index + /// space; max() until the first batch is read. + size_t previous_batch_start_pos_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; bool closed_ = false; }; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 3fc90514..95637720 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -46,9 +46,9 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara std::shared_ptr fs = std::make_shared(); ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/" + paimon_blob_file)); - ASSERT_OK_AND_ASSIGN(auto reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - blob_as_descriptor, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( + input_stream, /*batch_size=*/1024, blob_as_descriptor, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection_bitmap)); ASSERT_OK_AND_ASSIGN(auto chunked_array, paimon::test::ReadResultCollector::CollectResult(reader.get())); @@ -164,32 +164,71 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK_AND_ASSIGN( std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); ASSERT_EQ(3, number_of_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); ArrowArrayRelease(batch1.first.get()); ArrowSchemaRelease(batch1.second.get()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); - ASSERT_EQ(1, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(1, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch2.first.get()); ArrowSchemaRelease(batch2.second.get()); ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); - ASSERT_EQ(2, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch3.first.get()); ArrowSchemaRelease(batch3.second.get()); ASSERT_OK_AND_ASSIGN(auto batch4, reader->NextBatch()); - ASSERT_EQ(3, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); } +TEST_F(BlobFileBatchReaderTest, TestRowNumbersWithSelectionBitmap) { + // a selection bitmap removes rows, but batch positions must still map back to the original + // file row indexes so _ROW_ID completion works under row-range pushdown + auto schema = arrow::schema({BlobUtils::ToArrowField("my_blob_field", false)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + + std::string test_data_path = paimon::test::GetDataDir() + "/db_with_blob.db/table_with_blob/"; + auto dir = paimon::test::UniqueTestDirectory::Create(); + std::string table_path = dir->Str(); + ASSERT_TRUE(paimon::test::TestUtil::CopyDirectory(test_data_path, table_path)); + + std::shared_ptr fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr input_stream, + fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); + + RoaringBitmap32 selection; + selection.Add(0); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection)); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "No batch has been read yet"); + ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); + ArrowArrayRelease(batch1.first.get()); + ArrowSchemaRelease(batch1.second.get()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); + ArrowArrayRelease(batch2.first.get()); + ArrowSchemaRelease(batch2.second.get()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); + ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); + ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "Last batch was EOF"); + ASSERT_TRUE(BatchReader::IsEofBatch(batch3)); +} + TEST_F(BlobFileBatchReaderTest, InvalidScenario) { auto dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -205,20 +244,22 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) { { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(input_stream, - /*batch_size=*/0, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/0, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: read batch size '0' should be larger than zero"); } { ASSERT_NOK_WITH_MSG( BlobFileBatchReader::Create(/*input_stream=*/nullptr, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_), + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_), "blob file batch reader create failed: input stream is nullptr"); } { ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(/*input_stream=*/input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->GetFileSchema(), "blob file has no self-describing file schema"); ASSERT_TRUE(reader->GetReaderMetrics()); @@ -237,10 +278,11 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { file_system->Create(dir->Str() + "/file.blob", /*overwrite=*/true)); std::shared_ptr blob_field = BlobUtils::ToArrowField("blob_col"); auto struct_type = arrow::struct_({blob_field}); - bool blob_as_descriptor = GetParam(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor, output_stream, struct_type, - file_system, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -251,15 +293,15 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(dir->Str() + "/file.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); ASSERT_EQ(0, number_of_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch, reader->NextBatch()); ASSERT_TRUE(BatchReader::IsEofBatch(batch)); } @@ -277,9 +319,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "SetReadSchema failed: read schema cannot be nullptr"); @@ -302,9 +344,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "read schema field number 2 is not 1"); @@ -327,9 +369,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "field my_blob_field: large_binary is not BLOB"); @@ -350,9 +392,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); RoaringBitmap32 roaring; roaring.Add(0); roaring.Add(1); diff --git a/src/paimon/format/blob/blob_file_format_factory_test.cpp b/src/paimon/format/blob/blob_file_format_factory_test.cpp index 05898987..a5807fd6 100644 --- a/src/paimon/format/blob/blob_file_format_factory_test.cpp +++ b/src/paimon/format/blob/blob_file_format_factory_test.cpp @@ -18,8 +18,24 @@ #include "paimon/format/blob/blob_file_format_factory.h" +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/data/blob.h" +#include "paimon/defs.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/format_writer.h" +#include "paimon/format/writer_builder.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { @@ -31,4 +47,46 @@ TEST(BlobFileFormatFactoryTest, TestIdentifier) { ASSERT_EQ(file_format->Identifier(), "blob"); } +TEST(BlobFileFormatFactoryTest, TestWriteNullOptionPropagation) { + // Verifies the option flows through the production path + // FileFormatFactory::Get -> BlobFileFormat -> BlobWriterBuilder -> BlobFormatWriter. + std::unique_ptr dir = + paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr fs = std::make_shared(); + auto struct_type = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + + auto write_once = [&](const std::map& options, + const std::string& file_name) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get("blob", options)); + auto schema = arrow::schema(struct_type->fields()); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer_builder, + format->CreateWriterBuilder(&c_schema, /*batch_size=*/1024)); + // The blob writer builder is a SpecificFSWriterBuilder by construction. + static_cast(writer_builder.get())->WithFileSystem(fs); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, + fs->Create(dir->Str() + "/" + file_name, /*overwrite=*/true)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + writer_builder->Build(out, "none")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr array, + paimon::test::TestHelper::MakeBlobDescriptorArray( + struct_type, missing_blob, GetDefaultPool())); + ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array)); + return writer->Finish(); + }; + + // Without the option, writing the missing descriptor fails. + ASSERT_NOK_WITH_MSG(write_once({}, "no_option.blob"), "not exists"); + // The option set in the format options map reaches the writer. + ASSERT_OK(write_once({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}}, "with_option.blob")); +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index d9e8f4dc..c3666569 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -19,38 +19,51 @@ #include "paimon/format/blob/blob_format_writer.h" #include +#include +#include #include "arrow/api.h" #include "arrow/c/bridge.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/delta_varint_compressor.h" #include "paimon/data/blob.h" +#include "paimon/fs/file_system.h" #include "paimon/io/byte_array_input_stream.h" +#include "paimon/logging.h" namespace paimon::blob { -BlobFormatWriter::BlobFormatWriter(bool blob_as_descriptor, - const std::shared_ptr& out, +BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, + bool write_null_on_missing_file, + bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) - : blob_as_descriptor_(blob_as_descriptor), - out_(out), + : out_(out), + uri_(uri), data_type_(data_type), fs_(fs), - pool_(pool) { + pool_(pool), + write_null_on_missing_file_(write_null_on_missing_file), + write_null_on_fetch_failure_(write_null_on_fetch_failure), + write_placeholder_(write_placeholder) { + // Create() has already checked that data_type has exactly one BLOB field. + blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); + magic_number_bytes_ = IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); + logger_ = Logger::GetLogger("BlobFormatWriter"); } Result> BlobFormatWriter::Create( - bool blob_as_descriptor, const std::shared_ptr& out, - const std::shared_ptr& data_type, const std::shared_ptr& fs, - const std::shared_ptr& pool) { + const std::shared_ptr& out, const std::shared_ptr& data_type, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, + const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); } @@ -60,6 +73,9 @@ Result> BlobFormatWriter::Create( if (pool == nullptr) { return Status::Invalid("blob format writer create failed. pool is nullptr"); } + if (fs == nullptr) { + return Status::Invalid("blob format writer create failed. fs is nullptr"); + } if (data_type->num_fields() != 1) { return Status::Invalid( fmt::format("blob data type field number {} is not 1", data_type->num_fields())); @@ -68,8 +84,10 @@ Result> BlobFormatWriter::Create( return Status::Invalid( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); return std::unique_ptr( - new BlobFormatWriter(blob_as_descriptor, out, data_type, fs, pool)); + new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -103,13 +121,25 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { const auto& blob_array = arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); - PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); - + std::string_view blob_data = blob_array.GetView(0); + // Only a data-evolution partial-update write interprets the sentinel, which marks a row + // the update did not touch; any other write stores the bytes verbatim. See + // BlobDefs::kPlaceholderSentinel for the accepted collision with a sentinel-equal user + // value. + if (write_placeholder_ && BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) { + bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength); + return Status::OK(); + } + PAIMON_RETURN_NOT_OK(WriteBlob(blob_data)); PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } Status BlobFormatWriter::Flush() { + metrics_->SetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT, + null_on_missing_file_count_); + metrics_->SetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT, + null_on_fetch_failure_count_); return out_->Flush(); } @@ -131,36 +161,45 @@ Status BlobFormatWriter::Finish() { } Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - crc32_ = 0; - PAIMON_ASSIGN_OR_RAISE(int64_t previous_pos, out_->GetPos()); - - // write magic number - static PAIMON_UNIQUE_PTR kMagicNumberBytes = - IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); - PAIMON_RETURN_NOT_OK(WriteWithCrc32(kMagicNumberBytes->data(), kMagicNumberBytes->size())); - - // write blob content + // Open the blob input stream before writing any bytes, so that a failed fetch can be + // converted to a NULL element without leaving partial data in the output stream. + // Whether blob_data is a serialized BlobDescriptor is detected by its magic header rather + // than taken from a blob_as_descriptor option, so each row may hold either form. std::unique_ptr in; - if (blob_as_descriptor_) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, - Blob::FromDescriptor(blob_data.data(), blob_data.size())); - PAIMON_ASSIGN_OR_RAISE(in, blob->NewInputStream(fs_)); + PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, + BlobDescriptor::IsBlobDescriptor(blob_data.data(), blob_data.size())); + if (is_descriptor) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr descriptor_in, + OpenDescriptorInputStream(blob_data)); + // A null stream means a write-null option already converted the failure. + if (descriptor_in == nullptr) { + bin_lengths_.push_back(BlobDefs::kNullBinLength); + return Status::OK(); + } + in = std::move(descriptor_in); } else { in = std::make_unique(blob_data.data(), blob_data.size()); } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - uint64_t total_read_length = 0; - auto read_len = static_cast(std::min(file_length, tmp_buffer_->size())); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); + + crc32_ = 0; + PAIMON_ASSIGN_OR_RAISE(int64_t previous_pos, out_->GetPos()); + + // write magic number + PAIMON_RETURN_NOT_OK(WriteWithCrc32(magic_number_bytes_->data(), magic_number_bytes_->size())); + int64_t total_read_length = 0; + int64_t read_len = std::min(file_length, static_cast(tmp_buffer_->size())); while (read_len > 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); - if (static_cast(actual_read_len) != read_len) { - return Status::Invalid("actual read length {}, not match with expect length {}", - actual_read_len, read_len); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); + if (actual_read_len != read_len) { + return Status::Invalid( + fmt::format("actual read length {}, not match with expect length {}", + actual_read_len, read_len)); } PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); total_read_length += actual_read_len; - read_len = static_cast( - std::min(file_length - total_read_length, tmp_buffer_->size())); + read_len = + std::min(file_length - total_read_length, static_cast(tmp_buffer_->size())); } // write bin length @@ -180,16 +219,86 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { return Status::OK(); } -Status BlobFormatWriter::WriteBytes(const char* data, int32_t length) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual, out_->Write(data, length)); +Result> BlobFormatWriter::OpenDescriptorInputStream( + std::string_view blob_data) { + // A descriptor that cannot be deserialized is a fetch failure: the referenced data cannot be + // reached. Its URI is inside the unreadable bytes, hence the placeholder; the underlying + // status comes from the byte reader and never mentions blobs, hence the added context. + Result> blob_result = + Blob::FromDescriptor(blob_data.data(), blob_data.size()); + if (!blob_result.ok()) { + const Status& status = blob_result.status(); + return HandleFetchFailure( + "", status.WithMessage("invalid blob descriptor: ", status.message())); + } + std::unique_ptr blob = std::move(blob_result).value(); + + // A missing file is identified by FileSystem::Exists rather than by the status of a failed + // open, since file system implementations disagree on which status a missing file maps to. + // The check runs only when `write_null_on_missing_file_` needs the classification; otherwise + // a missing file gets the same treatment as any other failed open. + if (write_null_on_missing_file_) { + Result exists = fs_->Exists(blob->Uri()); + if (exists.ok()) { + if (!exists.value()) { + return HandleMissingFile(blob->Uri()); + } + } else if (!write_null_on_fetch_failure_) { + // The check cannot answer whether the file is there; with no fetch-failure handling + // to defer to, fail rather than assume either answer. + const Status& status = exists.status(); + return status.WithMessage("failed to check existence of blob file '", blob->Uri(), + "': ", status.message()); + } + // A failed check is otherwise deferred to the open below, which can still succeed. + } + + Result> opened = blob->NewInputStream(fs_); + if (!opened.ok()) { + // The file can be deleted between the check above and this open. Classifying that from + // `opened.status()` would reintroduce the plugin-specific status codes this writer avoids, + // so ask FileSystem::Exists once more. This narrows the window rather than closing it; a + // check that cannot answer falls through to the open failure. + if (write_null_on_missing_file_) { + Result exists = fs_->Exists(blob->Uri()); + if (exists.ok() && !exists.value()) { + return HandleMissingFile(blob->Uri()); + } + } + return HandleFetchFailure(blob->Uri(), opened.status()); + } + return std::move(opened).value(); +} + +std::unique_ptr BlobFormatWriter::HandleMissingFile(const std::string& blob_uri) { + PAIMON_LOG_WARN(logger_, "Blob file %s does not exist, writing NULL for BLOB field %s into %s", + blob_uri.c_str(), blob_field_name_.c_str(), uri_.c_str()); + ++null_on_missing_file_count_; + return std::unique_ptr(); +} + +Result> BlobFormatWriter::HandleFetchFailure( + const std::string& blob_uri, const Status& status) { + if (!write_null_on_fetch_failure_) { + return status; + } + PAIMON_LOG_WARN(logger_, "Failed to fetch blob %s, writing NULL for BLOB field %s into %s: %s", + blob_uri.c_str(), blob_field_name_.c_str(), uri_.c_str(), + status.ToString().c_str()); + ++null_on_fetch_failure_count_; + return std::unique_ptr(); +} + +Status BlobFormatWriter::WriteBytes(const char* data, int64_t length) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual, out_->Write(data, length)); if (actual != length) { - return Status::Invalid("not suppose actual length {} not match with expect {}", actual, - length); + return Status::Invalid( + fmt::format("unexpected actual length {} not match with expect {}", actual, length)); } return Status::OK(); } -Status BlobFormatWriter::WriteWithCrc32(const char* data, int32_t length) { +Status BlobFormatWriter::WriteWithCrc32(const char* data, int64_t length) { crc32_ = arrow::internal::crc32(crc32_, data, length); return WriteBytes(data, length); } @@ -199,6 +308,10 @@ Result BlobFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return current_pos >= target_size; } +Status BlobFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by blob format writer."); +} + template PAIMON_UNIQUE_PTR BlobFormatWriter::IntegerToLittleEndian( T value, const std::shared_ptr& pool) { diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 542fe26c..9537b347 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -19,13 +19,16 @@ #pragma once #include +#include #include +#include #include #include #include "arrow/api.h" #include "arrow/util/crc32.h" #include "paimon/format/format_writer.h" +#include "paimon/logging.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -39,20 +42,44 @@ struct ArrowArray; namespace paimon { class Blob; class FileSystem; +class InputStream; class Metrics; class OutputStream; } // namespace paimon namespace paimon::blob { +class BlobMetrics { + public: + /// Number of rows written as NULL because their referenced file did not exist. + static inline const char WRITE_NULL_ON_MISSING_FILE_COUNT[] = + "blob.write.null-on-missing-file.count"; + /// Number of rows written as NULL because their referenced data could not be reached. + static inline const char WRITE_NULL_ON_FETCH_FAILURE_COUNT[] = + "blob.write.null-on-fetch-failure.count"; +}; + // Blob format: // https://cwiki.apache.org/confluence/display/PAIMON/PIP-35%3A+Introduce+Blob+to+store+multimodal+data class BlobFormatWriter : public FormatWriter { public: + /// `write_null_on_missing_file` converts a descriptor whose referenced file does not exist + /// (as reported by FileSystem::Exists) to a NULL element, and `write_null_on_fetch_failure` + /// converts any other failure to access the referenced data; failures during the streaming + /// copy always fail the write. The existence check runs only when + /// `write_null_on_missing_file` is enabled; otherwise a missing file follows + /// `write_null_on_fetch_failure` like any other failed open. + /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// + /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a + /// data-evolution partial update) persists a value exactly equal to + /// BlobDefs::kPlaceholderSentinel as a placeholder entry (bin_length -2, no data bytes); + /// any other value is stored verbatim. When disabled, values are never interpreted and can + /// never be turned into placeholder entries. static Result> Create( - bool blob_as_descriptor, const std::shared_ptr& out, - const std::shared_ptr& data_type, const std::shared_ptr& fs, - const std::shared_ptr& pool); + const std::shared_ptr& out, const std::shared_ptr& data_type, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, + const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -66,34 +93,60 @@ class BlobFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: - BlobFormatWriter(bool blob_as_descriptor, const std::shared_ptr& out, + BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, - const std::shared_ptr& fs, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); - Status WriteBytes(const char* data, int32_t length); - Status WriteWithCrc32(const char* data, int32_t length); + /// Deserialize the descriptor and open an input stream on the referenced data. + /// Returns a null stream when the failure is converted to a NULL element by + /// `write_null_on_missing_file_` or `write_null_on_fetch_failure_`. + Result> OpenDescriptorInputStream(std::string_view blob_data); + + /// Convert a file that FileSystem::Exists reported as absent to a NULL element: count it and + /// return a null stream. Only reached under `write_null_on_missing_file_`, which callers check. + std::unique_ptr HandleMissingFile(const std::string& blob_uri); + + /// Apply `write_null_on_fetch_failure_` to a failure to reach the referenced data: returns + /// `status` when the option is disabled, and a null stream when it converts the failure to a + /// NULL element. `blob_uri` is only used for logging. + Result> HandleFetchFailure(const std::string& blob_uri, + const Status& status); + + Status WriteBytes(const char* data, int64_t length); + Status WriteWithCrc32(const char* data, int64_t length); template static PAIMON_UNIQUE_PTR IntegerToLittleEndian(T value, const std::shared_ptr& pool); - public: + private: static constexpr uint32_t kTmpBufferSize = 1024 * 1024; - private: - bool blob_as_descriptor_; uint32_t crc32_ = 0; std::vector bin_lengths_; std::shared_ptr out_; + /// Path of the blob file being written, not of any referenced blob. + std::string uri_; PAIMON_UNIQUE_PTR tmp_buffer_; + PAIMON_UNIQUE_PTR magic_number_bytes_; std::shared_ptr data_type_; + std::string blob_field_name_; std::shared_ptr fs_; std::shared_ptr pool_; std::shared_ptr metrics_; + bool write_null_on_missing_file_ = false; + bool write_null_on_fetch_failure_ = false; + bool write_placeholder_ = false; + uint64_t null_on_missing_file_count_ = 0; + uint64_t null_on_fetch_failure_count_ = 0; + std::unique_ptr logger_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 99022259..16c9e2af 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -20,9 +20,11 @@ #include #include +#include #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/data/blob.h" @@ -33,10 +35,84 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { -class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParamInterface { + +/// A file system whose Open() always fails with the configured status while Exists() keeps the +/// real local check, standing in for a plugin that reports a missing file as something other than +/// Status::NotExist. Open() calls are counted so a test can assert a missing file is never opened. +class OpenFailFileSystem : public LocalFileSystem { + public: + explicit OpenFailFileSystem(Status open_status) : open_status_(std::move(open_status)) {} + + Result> Open(const std::string& path) const override { + ++open_call_count_; + return open_status_; + } + + int64_t OpenCallCount() const { + return open_call_count_; + } + + private: + Status open_status_; + mutable int64_t open_call_count_ = 0; +}; + +/// A file system whose Exists() always fails with the configured status, counting the calls. By +/// default Open() is delegated to a separate LocalFileSystem so that it still succeeds +/// (LocalFileSystem::Open() calls Exists() on itself, so without the delegation a failed check +/// would also fail the open); a non-OK `open_status` makes Open() fail with it instead. +class ExistsFailFileSystem : public LocalFileSystem { + public: + explicit ExistsFailFileSystem(Status exists_status, Status open_status = Status::OK()) + : exists_status_(std::move(exists_status)), open_status_(std::move(open_status)) {} + + Result Exists(const std::string& path) const override { + ++exists_call_count_; + return exists_status_; + } + + Result> Open(const std::string& path) const override { + if (!open_status_.ok()) { + return open_status_; + } + return real_fs_.Open(path); + } + + int64_t ExistsCallCount() const { + return exists_call_count_; + } + + private: + Status exists_status_; + Status open_status_; + LocalFileSystem real_fs_; + mutable int64_t exists_call_count_ = 0; +}; + +/// A file system that reports a file as present on the first Exists() and absent afterwards, +/// standing in for a file deleted between the check and the open. Open() fails with a plain +/// IOError, so a test can tell a re-checked classification apart from one taken from the open. +class VanishingFileSystem : public LocalFileSystem { + public: + Result Exists(const std::string& path) const override { + return ++exists_call_count_ == 1; + } + + Result> Open(const std::string& path) const override { + return Status::IOError("mock io error"); + } + + int64_t ExistsCallCount() const { + return exists_call_count_; + } + + private: + mutable int64_t exists_call_count_ = 0; +}; + +class BlobFormatWriterTestBase : public ::testing::Test { public: void SetUp() override { - blob_as_descriptor_ = GetParam(); pool_ = GetDefaultPool(); dir_ = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); @@ -51,35 +127,68 @@ class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParam ASSERT_OK(output_stream_->Close()); } - Result> PrepareBlobArray( + /// Create a writer on output_stream_ with both write-null options disabled. + Result> CreateDefaultWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_); + } + + /// Create a writer in placeholder mode, as used by data-evolution partial updates. + Result> CreatePlaceholderWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system_, pool_); + } + + Status AddBatchOnce(const std::shared_ptr& format_writer, + const std::shared_ptr& blob_array) const { + auto c_array = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get())); + return format_writer->AddBatch(c_array.get()); + } + + Result> PrepareDescriptorArray( const std::shared_ptr& blob) const { + return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_); + } + + /// Build a single-row blob array holding `bytes` verbatim, bypassing the Blob helpers. + Result> MakeBlobArrayFromBytes(const std::string& bytes) const { arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), {std::make_shared()}); auto blob_builder = static_cast(struct_builder.field_builder(0)); PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); - if (blob_as_descriptor_) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append( - blob->ToDescriptor(pool_)->data(), blob->ToDescriptor(pool_)->size())); - } else { - PAIMON_ASSIGN_OR_RAISE(auto blob_data, blob->ToData(file_system_, pool_)); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - blob_builder->Append(blob_data->data(), blob_data->size())); - } + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append(bytes.data(), bytes.size())); std::shared_ptr array; PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); return array; } - Status AddBatchOnce(const std::shared_ptr& format_writer, - const std::shared_ptr& blob_array) const { - auto c_array = std::make_unique(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get())); - return format_writer->AddBatch(c_array.get()); + Result> ReadBackAsData() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); + PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(chunked_array->chunks())); + return arrow::internal::checked_pointer_cast(concat_array); } - private: - bool blob_as_descriptor_; + protected: std::shared_ptr pool_; std::unique_ptr dir_; std::shared_ptr output_stream_; @@ -87,13 +196,45 @@ class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParam std::shared_ptr struct_type_; }; +class BlobFormatWriterTest : public BlobFormatWriterTestBase, + public ::testing::WithParamInterface { + public: + void SetUp() override { + blob_as_descriptor_ = GetParam(); + BlobFormatWriterTestBase::SetUp(); + } + + Result> PrepareBlobArray( + const std::shared_ptr& blob) const { + if (blob_as_descriptor_) { + return PrepareDescriptorArray(blob); + } + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); + PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR blob_data, + blob->ToData(file_system_, pool_)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append(blob_data->data(), blob_data->size())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); + return array; + } + + private: + bool blob_as_descriptor_; +}; + +/// The write-null tests always feed descriptor bytes, so they do not depend on the +/// blob_as_descriptor_ parameter and run once on the non-parameterized fixture. +using BlobFormatWriterWriteNullTest = BlobFormatWriterTestBase; + INSTANTIATE_TEST_SUITE_P(BlobAsDescriptor, BlobFormatWriterTest, ::testing::Values(false, true)); TEST_P(BlobFormatWriterTest, TestSimple) { // write - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; @@ -124,7 +265,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -154,38 +296,53 @@ TEST_P(BlobFormatWriterTest, TestSimple) { TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(blob_as_descriptor_, nullptr, struct_type_, file_system_, pool_), + BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, nullptr, file_system_, pool_), + BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, nullptr), - "blob format writer create failed. pool is nullptr"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, nullptr), + "blob format writer create failed. pool is nullptr"); + + // Test with nullptr file system + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, nullptr, pool_), + "blob format writer create failed. fs is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, - multi_field_type, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, non_blob_type, - file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "field regular_col: binary is not BLOB"); } TEST_P(BlobFormatWriterTest, TestInvalidCase) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Test nullptr batch ASSERT_NOK_WITH_MSG(writer->AddBatch(nullptr), @@ -202,9 +359,7 @@ TEST_P(BlobFormatWriterTest, TestInvalidCase) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Test batch with wrong length (not 1) arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -230,9 +385,7 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { } TEST_P(BlobFormatWriterTest, TestReachTargetSize) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Initially should not reach target size ASSERT_OK_AND_ASSIGN(bool reached, writer->ReachTargetSize(true, 1000)); @@ -255,9 +408,7 @@ TEST_P(BlobFormatWriterTest, TestReachTargetSize) { } TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); auto metrics = writer->GetWriterMetrics(); ASSERT_TRUE(metrics); @@ -265,9 +416,7 @@ TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { TEST_P(BlobFormatWriterTest, TestEmptyWriter) { // Test creating a writer and finishing without adding any data - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -276,7 +425,7 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t file_length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); ASSERT_EQ(file_length, 5); // Should have footer even if no data std::vector buffer(file_length); ASSERT_OK_AND_ASSIGN(auto read_length, input_stream->Read(buffer.data(), buffer.size())); @@ -286,9 +435,7 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { } TEST_P(BlobFormatWriterTest, TestLargeBlob) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Create a temporary large file for testing std::string large_file_path = dir_->Str() + "/large_test_file.bin"; @@ -298,7 +445,7 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { // Write data larger than TMP_BUFFER_SIZE (1MB) const size_t large_size = BlobFormatWriter::kTmpBufferSize * 2 + 1000; // ~2MB std::vector large_data(large_size, 'A'); - ASSERT_OK_AND_ASSIGN(int32_t written, large_file_stream->Write(large_data.data(), large_size)); + ASSERT_OK_AND_ASSIGN(int64_t written, large_file_stream->Write(large_data.data(), large_size)); ASSERT_EQ(written, large_size); ASSERT_OK(large_file_stream->Flush()); ASSERT_OK(large_file_stream->Close()); @@ -315,7 +462,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -341,9 +489,7 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Write one row with child-level null blob arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -366,7 +512,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -389,18 +536,406 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(struct_builder2.Finish(&null_struct_array).ok()); auto null_c_array = std::make_unique(); ASSERT_TRUE(arrow::ExportArray(*null_struct_array, null_c_array.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer2, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer2, CreateDefaultWriter()); ASSERT_NOK_WITH_MSG(writer2->AddBatch(null_c_array.get()), "BlobFormatWriter does not support struct-level null."); ArrowArrayRelease(null_c_array.get()); } +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + + // A fetch failure is not converted to NULL by write_null_on_missing_file alone; the + // rejected row leaves the writer usable. + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, bad_offset_array), "exceed total length"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, Blob::FromPath(file)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + ASSERT_OK(AddBatchOnce(writer, array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 2); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_FALSE(result_struct->field(0)->IsNull(1)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(1), + std::string_view(expected_data->data(), expected_data->size())); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); + + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_OK(AddBatchOnce(writer, bad_offset_array)); + + // Without write_null_on_missing_file no existence check runs, so a missing file is not told + // apart from any other failed open and is converted by this option. + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Both rows count as fetch failures. + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 2); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 2); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_TRUE(result_struct->field(0)->IsNull(1)); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); + + // Row 0: missing file -> NULL. + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + + // Row 1: fetch failure (offset beyond EOF) -> NULL. + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_OK(AddBatchOnce(writer, bad_offset_array)); + + // Row 2: valid blob. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, Blob::FromPath(file)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + ASSERT_OK(AddBatchOnce(writer, array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // The two NULL rows had different causes, counted separately. + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 3); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_TRUE(result_struct->field(0)->IsNull(1)); + ASSERT_FALSE(result_struct->field(0)->IsNull(2)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(2), + std::string_view(expected_data->data(), expected_data->size())); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { + // Each case needs its own option pair and therefore its own writer, so none of them finishes + // the shared output stream: this test only asserts how a failure is classified. That the + // resulting NULL element is written correctly is covered by the TestWriteNullOn* tests. + auto io_error_fs = std::make_shared(Status::IOError("mock io error")); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr existing_blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + ASSERT_OK_AND_ASSIGN(auto existing_array, PrepareDescriptorArray(existing_blob)); + + // Missing file: classified without opening it, so what Open would return is irrelevant. + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + ASSERT_EQ(io_error_fs->OpenCallCount(), 0); + } + // Existing file that cannot be opened: a fetch failure, which this writer does not convert. + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); + } + // The same fetch failure, now converted to NULL. + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + } + // Missing file with only fetch-failure enabled: no existence check runs, so the file is + // opened and the failure is converted like any other fetch failure. The mock's count + // accumulates across cases, so compare against it. + { + const int64_t open_calls_before = io_error_fs->OpenCallCount(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + } + + // A file deleted between the check and the open is still a missing file: the failed open + // triggers one more check rather than being classified by its status. Without it the deletion + // would defeat write_null_on_missing_file, which does not convert a fetch failure. Each case + // needs its own file system, since the mock reports the file as present only on the first call. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + // One check before the open and one after it. + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + // The same deletion with both options enabled: classified as missing rather than swallowed + // by fetch-failure. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + // The same deletion with only fetch-failure enabled: existence is never consulted, and the + // failed open is converted like any other fetch failure. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); + } + // Neither option: existence is never consulted, and the open failure propagates as it is. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); + } +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { + // Descriptor detection only inspects version and magic, so a descriptor truncated after those + // passes detection and then fails to deserialize: a fetch failure, not a missing file. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool_); + ASSERT_GT(descriptor->size(), 8); + std::string truncated(descriptor->data(), descriptor->size() - 8); + ASSERT_OK_AND_ASSIGN(bool is_descriptor, + BlobDescriptor::IsBlobDescriptor(truncated.data(), truncated.size())); + ASSERT_TRUE(is_descriptor); + ASSERT_OK_AND_ASSIGN(auto array, MakeBlobArrayFromBytes(truncated)); + + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); + } + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + } + + // Only the second case reaches the stream; the first fails before writing any byte. + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 1); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { + // An existence check that cannot answer leaves it unknown whether the file is there. With no + // fetch-failure handling to defer to, the write fails; otherwise the failed check is deferred + // to the open, whose own outcome decides. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + + // Deferred check failure whose open succeeds: the blob is written as data, not as NULL. + // This case finishes the shared output stream, so it runs first and is read back below. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 1); + ASSERT_FALSE(result_struct->field(0)->IsNull(0)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(0), + std::string_view(expected_data->data(), expected_data->size())); + + // With no fetch-failure handling to defer to, the check failure fails the write. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, exists_fail_fs, pool_)); + // The reported failure names the check and keeps the underlying status message. + Status check_status = AddBatchOnce(writer, array); + ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); + ASSERT_NOK_WITH_MSG(check_status, "mock exists error"); + } + // Deferred check failure whose open then fails: a fetch failure. The re-check after the + // failed open cannot answer either, so it falls through to the open failure. + { + auto exists_fail_fs = std::make_shared( + Status::IOError("mock exists error"), Status::IOError("mock open error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + // One check before the open and one after it failed. + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + } + // With only write_null_on_fetch_failure, no existence check runs at all; the open succeeds + // and the blob is written as data. A separate output stream keeps the data bytes out of the + // already finished shared stream. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr side_stream, + file_system_->Create(dir_->Str() + "/side.blob", /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + side_stream, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + ASSERT_OK(side_stream->Flush()); + ASSERT_OK(side_stream->Close()); + } +} + TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Create a zero-length file std::string zero_file_path = dir_->Str() + "/zero_length_file.bin"; @@ -422,7 +957,7 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t file_length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); ASSERT_EQ(file_length, 22); std::vector buffer(file_length); ASSERT_OK_AND_ASSIGN(auto read_length, @@ -434,4 +969,238 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_EQ(buffer, expected); } +/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so +/// they do not depend on the blob_as_descriptor_ parameter and run once on the +/// non-parameterized fixture. +using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase; + +std::string PlaceholderSentinelBytes() { + return std::string(BlobDefs::PlaceholderSentinelView()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + + // row 0: inline bytes "inline"; row 1: null; row 2: placeholder + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Verify byte-level alignment with the Java writer (BlobFormatWriterTest + // testRawBlobGoldenBytes): null and placeholder rows occupy no data bytes; the index + // records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01]. + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); + ASSERT_EQ(file_length, 30); + std::vector buffer(file_length); + ASSERT_OK_AND_ASSIGN(auto read_length, + input_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_EQ(read_length, 30); + std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32 + 0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60, + 0xc8, 0xe9, + // index of [22, -1, -2] + 0x2c, 0x2d, 0x01, + // footer: index length + version + 0x03, 0x00, 0x00, 0x00, 0x01}}; + ASSERT_EQ(buffer, expected); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // default (strict) mode: reading a placeholder entry fails + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder"); + } + + // placeholder-aware mode: the entry is returned as the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), "inline"); + ASSERT_FALSE(binary_array->IsNull(1)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } + + // placeholder-aware descriptor mode also returns the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first")); + ASSERT_OK(AddBatchOnce(writer, array0)); + ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, array1)); + ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third")); + ASSERT_OK(AddBatchOnce(writer, array2)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + RoaringBitmap32 selection; + selection.Add(1); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); + ASSERT_EQ(binary_array->GetString(1), "third"); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) { + // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it + // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); + ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, sentinel_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 1); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(binary_array->IsNull(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelPrefixedValueVerbatimInPlaceholderMode) { + // placeholders are identified by exact equality only: even in placeholder mode a real + // value that merely starts with the sentinel bytes is stored verbatim and read back + // unchanged in both strict and placeholder-aware modes + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + std::string sentinel = PlaceholderSentinelBytes(); + ASSERT_OK_AND_ASSIGN(auto doubled_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel)); + ASSERT_OK(AddBatchOnce(writer, doubled_sentinel_array)); + ASSERT_OK_AND_ASSIGN(auto prefixed_array, MakeBlobArrayFromBytes(sentinel + "suffix")); + ASSERT_OK(AddBatchOnce(writer, prefixed_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + for (bool emit_placeholder_sentinel : {false, true}) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + emit_placeholder_sentinel, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(struct_array->length(), 2); + ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h index 85bebe96..46bb1d25 100644 --- a/src/paimon/format/blob/blob_reader_builder.h +++ b/src/paimon/format/blob/blob_reader_builder.h @@ -45,7 +45,11 @@ class BlobReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( bool blob_as_descriptor, OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_); + PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel, + OptionsUtils::GetValueFromMap( + options_, BlobDefs::kEmitPlaceholderSentinelKey, false)); + return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, + emit_placeholder_sentinel, pool_); } private: diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index 12a7abdd..ca671e4e 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -50,13 +50,16 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ fmt::format("field {} is not BLOB", write_schema_->field(0)->ToString())); } + // The reader only serves footer metadata (GetNumberOfRows); NextBatch is never called, so + // the strict placeholder mode is irrelevant even for files containing placeholder entries. PAIMON_ASSIGN_OR_RAISE( std::unique_ptr blob_reader, BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); + /*batch_size=*/1024, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool)); ColumnStatsVector result_stats; result_stats.push_back( - ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, /*null_count=*/0)); + ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); PAIMON_ASSIGN_OR_RAISE(uint64_t num_rows, blob_reader->GetNumberOfRows()); return std::make_pair(result_stats, FileInfo(num_rows)); } diff --git a/src/paimon/format/blob/blob_stats_extractor_test.cpp b/src/paimon/format/blob/blob_stats_extractor_test.cpp index e6c8d694..befbfa52 100644 --- a/src/paimon/format/blob/blob_stats_extractor_test.cpp +++ b/src/paimon/format/blob/blob_stats_extractor_test.cpp @@ -70,7 +70,7 @@ TEST_F(BlobStatsExtractorTest, TestDifferentBlobFiles) { ASSERT_EQ(1u, stats_with_info.first.size()); ASSERT_TRUE(stats_with_info.first[0]); ASSERT_EQ(FieldType::STRING, stats_with_info.first[0]->GetFieldType()); - ASSERT_EQ("min null, max null, null count 0", stats_with_info.first[0]->ToString()); + ASSERT_EQ("min null, max null, null count null", stats_with_info.first[0]->ToString()); // Check row count matches expected ASSERT_EQ(expected_rows, stats_with_info.second.GetRowCount()) diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 1fc1a58e..55254534 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -26,6 +26,7 @@ #include #include "arrow/api.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/utils/options_utils.h" #include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" @@ -69,10 +70,17 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { if (fs_ == nullptr) { return Status::Invalid("File system is nullptr. Please call WithFileSystem() first."); } + PAIMON_ASSIGN_OR_RAISE(bool write_null_on_missing_file, + OptionsUtils::GetValueFromMap( + options_, Options::BLOB_WRITE_NULL_ON_MISSING_FILE, false)); + PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, + OptionsUtils::GetValueFromMap( + options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); PAIMON_ASSIGN_OR_RAISE( - bool blob_as_descriptor, - OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFormatWriter::Create(blob_as_descriptor, out, data_type_, fs_, pool_); + bool write_placeholder, + OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false)); + return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs_, pool_); } private: diff --git a/src/paimon/format/blob/blob_writer_builder_test.cpp b/src/paimon/format/blob/blob_writer_builder_test.cpp index 2350eba2..5b522970 100644 --- a/src/paimon/format/blob/blob_writer_builder_test.cpp +++ b/src/paimon/format/blob/blob_writer_builder_test.cpp @@ -19,10 +19,15 @@ #include "paimon/format/blob/blob_writer_builder.h" #include "arrow/api.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/data/blob.h" +#include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { @@ -34,7 +39,7 @@ class BlobWriterBuilderTest : public ::testing::Test { file_system_ = std::make_shared(); ASSERT_OK_AND_ASSIGN(output_stream_, file_system_->Create(dir_->Str() + "/file.blob", /*overwrite=*/true)); - struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", false)}); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); } void TearDown() override {} @@ -54,4 +59,88 @@ TEST_F(BlobWriterBuilderTest, TestSimple) { ASSERT_OK(builder.Build(output_stream_, "none")); } +TEST_F(BlobWriterBuilderTest, TestWriteNullOptions) { + auto prepare_batch = [&](const std::shared_ptr& blob, ArrowArray* c_array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr array, + paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, + GetDefaultPool())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array)); + return Status::OK(); + }; + // Each writer gets its own output file so that a Finish() in one scenario cannot leak + // bytes into the next. + int32_t file_index = 0; + auto build_writer = [&](const std::map& options) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr out, + file_system_->Create(dir_->Str() + "/file" + std::to_string(file_index++) + ".blob", + /*overwrite=*/true)); + BlobWriterBuilder builder(struct_type_, options); + builder.WithFileSystem(file_system_); + return builder.Build(out, "none"); + }; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + std::string data_file = dir_->Str() + "/data_file.bin"; + ASSERT_OK_AND_ASSIGN(auto data_file_stream, + file_system_->Create(data_file, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, data_file_stream->Write("blob data", 9)); + ASSERT_EQ(written, 9); + ASSERT_OK(data_file_stream->Flush()); + ASSERT_OK(data_file_stream->Close()); + // The offset beyond the end of the 9-byte file makes the descriptor a fetch failure. + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(data_file, /*offset=*/100, /*length=*/10)); + + // Both options default to false. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, build_writer({})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "not exists"); + } + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, build_writer({})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(bad_offset_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "exceed total length"); + } + + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK(writer->Finish()); + } + + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(bad_offset_blob, &c_array)); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK(writer->Finish()); + } + + // An explicit "false" exercises value parsing, not just the absent-key default. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "false"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "false"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "not exists"); + } + + { + ASSERT_NOK_WITH_MSG(build_writer({{Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "invalid"}}), + "convert key blob-write-null-on-fetch-failure"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/orc/CMakeLists.txt b/src/paimon/format/orc/CMakeLists.txt index 25b44a5a..d86750ea 100644 --- a/src/paimon/format/orc/CMakeLists.txt +++ b/src/paimon/format/orc/CMakeLists.txt @@ -40,7 +40,6 @@ if(PAIMON_ENABLE_ORC) fmt orc::orc tbb - dl Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -63,10 +62,8 @@ if(PAIMON_ENABLE_ORC) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_orc_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN} EXTRA_LINK_LIBS orc::orc) diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 5f76419f..992387d1 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -144,7 +144,9 @@ class UnPooledBooleanBuilder : public EmptyBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status SetData(const uint8_t* data, int64_t length) { @@ -189,7 +191,9 @@ class UnPooledPrimitiveBuilder : public arrow::NumericBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -230,7 +234,9 @@ class UnPooledLargeBinaryBuilder : public arrow::LargeBinaryBuilder { length_ += length; } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { std::shared_ptr null_bitmap; @@ -275,7 +281,9 @@ class UnPooledBinaryBuilder : public arrow::BinaryBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -315,7 +323,9 @@ class UnPooledListBuilder : public EmptyBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } void SetOffsets(const std::shared_ptr& offsets) { @@ -357,13 +367,15 @@ class UnPooledStructBuilder : public EmptyBuilder { arrow::FieldVector fields; fields.reserve(children_.size()); for (size_t i = 0; i < children_.size(); i++) { - fields.emplace_back(arrow::field(type_->field(i)->name(), children_[i]->type())); + fields.emplace_back(type_->field(i)->WithType(children_[i]->type())); } return arrow::struct_(fields); } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -489,7 +501,9 @@ class UnPooledStringDictionaryBuilder : public EmptyBuilder { indices_ = indices; } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { std::shared_ptr null_bitmap; @@ -639,19 +653,26 @@ Result> MakeOrcBackedTimestampBuilder( : nullptr; const int64_t* seconds = typed_batch->data.data(); const int64_t* nanos = typed_batch->nanoseconds.data(); + const bool has_nulls = typed_batch->hasNulls; + const auto* not_null = typed_batch->notNull.data(); + auto is_null = [has_nulls, not_null](int64_t index) { return has_nulls && !not_null[index]; }; auto timestamp_type = arrow::internal::checked_pointer_cast(type); assert(timestamp_type); int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); // TODO(lisizhuo.lsz): check nano overflow in arrow if (precision == Timestamp::MIN_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds](int64_t index) { return seconds[index]; }, typed_batch->numElements); + [seconds, is_null](int64_t index) { return is_null(index) ? 0 : seconds[index]; }, + typed_batch->numElements); PAIMON_RETURN_NOT_OK_FROM_ARROW( builder->AppendValues(transform_iter.begin(), transform_iter.end(), valid_bytes)); return builder; } else if (precision == Timestamp::MILLIS_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::MILLISECOND] + nanos[index] / @@ -663,7 +684,10 @@ Result> MakeOrcBackedTimestampBuilder( return builder; } else if (precision == Timestamp::DEFAULT_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::MICROSECOND] + nanos[index] / @@ -675,7 +699,10 @@ Result> MakeOrcBackedTimestampBuilder( return builder; } else if (precision == Timestamp::MAX_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::NANOSECOND] + nanos[index]; @@ -940,7 +967,7 @@ arrow::Result> NormalizeArray( auto child_length = struct_array->data()->child_data[i]->length; auto child_offset = struct_array->data()->child_data[i]->offset; // field function will change length & offset in child data - std::shared_ptr child = struct_array->field(static_cast(i)); + std::shared_ptr child = struct_array->field(static_cast(i)); const std::shared_ptr child_bitmap = child->null_bitmap(); std::shared_ptr final_child_bitmap; if (child_bitmap == nullptr && bitmap == nullptr) { @@ -1214,7 +1241,7 @@ arrow::Status WriteStructBatch(const arrow::Array& array, for (std::size_t i = 0; i < size; i++) { batch->fields[i]->resize(arrow_length); ARROW_RETURN_NOT_OK( - WriteBatch(*(struct_array->field(static_cast(i))), batch->fields[i])); + WriteBatch(*(struct_array->field(static_cast(i))), batch->fields[i])); } return arrow::Status::OK(); } @@ -1316,6 +1343,9 @@ arrow::Status WriteBatch(const arrow::Array& array, ::orc::ColumnVectorBatch* co case arrow::Type::type::BINARY: return WriteGenericBatch( array, column_vector_batch); + case arrow::Type::type::LARGE_BINARY: + return WriteGenericBatch( + array, column_vector_batch); case arrow::Type::type::STRING: return WriteGenericBatch( array, column_vector_batch); @@ -1379,6 +1409,7 @@ arrow::Result> GetOrcType(const arrow::DataType& ty case arrow::Type::type::STRING: return ::orc::createPrimitiveType(::orc::TypeKind::STRING); case arrow::Type::type::BINARY: + case arrow::Type::type::LARGE_BINARY: return ::orc::createPrimitiveType(::orc::TypeKind::BINARY); case arrow::Type::type::DATE32: return ::orc::createPrimitiveType(::orc::TypeKind::DATE); diff --git a/src/paimon/format/orc/orc_adapter_test.cpp b/src/paimon/format/orc/orc_adapter_test.cpp index 2678a286..1a43de5f 100644 --- a/src/paimon/format/orc/orc_adapter_test.cpp +++ b/src/paimon/format/orc/orc_adapter_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -184,12 +185,13 @@ TEST_F(OrcAdapterTest, TestGetOrcType) { auto col21_field = arrow::field("col21", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)); auto col22_field = arrow::field("col22", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)); auto col23_field = arrow::field("col23", arrow::timestamp(arrow::TimeUnit::NANO, timezone)); + auto col24_field = arrow::field("col24", arrow::large_binary()); auto arrow_schema = std::make_shared(arrow::FieldVector( {col1_field, col2_field, col3_field, col4_field, col5_field, col6_field, col7_field, col8_field, col9_field, col10_field, col11_field, col12_field, col13_field, col14_field, col15_field, col16_field, col17_field, col18_field, - col19_field, col20_field, col21_field, col22_field, col23_field})); + col19_field, col20_field, col21_field, col22_field, col23_field, col24_field})); ASSERT_OK_AND_ASSIGN(std::unique_ptr<::orc::Type> orc_type, OrcAdapter::GetOrcType(*arrow_schema)); ASSERT_TRUE(orc_type); @@ -199,7 +201,7 @@ TEST_F(OrcAdapterTest, TestGetOrcType) { "array,col14:map,col15:timestamp,col16:struct,col17:timestamp,col18:timestamp,col19:timestamp,col20:timestamp " "with local time zone,col21:timestamp with local time zone,col22:timestamp with local time " - "zone,col23:timestamp with local time zone>", + "zone,col23:timestamp with local time zone,col24:binary>", orc_type->toString()); } @@ -209,11 +211,6 @@ TEST_F(OrcAdapterTest, TestGetOrcTypeWithInvalidArrowType) { auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); ASSERT_NOK(OrcAdapter::GetOrcType(*arrow_schema)); } - { - auto col1_field = arrow::field("col1", arrow::large_binary()); - auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); - ASSERT_NOK(OrcAdapter::GetOrcType(*arrow_schema)); - } { auto col1_field = arrow::field("col1", arrow::uint32()); auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); @@ -570,6 +567,39 @@ TEST_P(OrcAdapterTest, TestAppendBatchWithBinaryForAllNull) { ASSERT_TRUE(converted_array->Equals(src_array)) << converted_array->ToString(); } +TEST_P(OrcAdapterTest, TestWriteBatchWithLargeBinary) { + arrow::FieldVector fields = {arrow::field("f0", arrow::large_binary())}; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])") + .ValueOrDie()); + + auto [orc_reader_holder, read_batch] = GenerateOrcReadBatch(src_array); + auto* struct_batch = dynamic_cast<::orc::StructVectorBatch*>(read_batch.get()); + ASSERT_TRUE(struct_batch); + ASSERT_EQ(1, struct_batch->fields.size()); + + auto* large_binary_batch = dynamic_cast<::orc::StringVectorBatch*>(struct_batch->fields[0]); + ASSERT_TRUE(large_binary_batch); + ASSERT_EQ(4, large_binary_batch->numElements); + + std::vector expected_values = {"descriptor-1", "", "descriptor-2"}; + ASSERT_TRUE(large_binary_batch->notNull[0]); + ASSERT_EQ(expected_values[0], + std::string(large_binary_batch->data[0], large_binary_batch->length[0])); + ASSERT_TRUE(large_binary_batch->notNull[1]); + ASSERT_EQ(expected_values[1], + std::string(large_binary_batch->data[1], large_binary_batch->length[1])); + ASSERT_FALSE(large_binary_batch->notNull[2]); + ASSERT_TRUE(large_binary_batch->notNull[3]); + ASSERT_EQ(expected_values[2], + std::string(large_binary_batch->data[3], large_binary_batch->length[3])); +} + TEST_P(OrcAdapterTest, TestDecimalAndTimestamp) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp index ed25327b..cd627eb5 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader.cpp @@ -26,9 +26,13 @@ #include #include "arrow/c/bridge.h" +#include "arrow/io/memory.h" +#include "arrow/ipc/api.h" +#include "arrow/util/base64.h" #include "fmt/format.h" #include "orc/OrcFile.hh" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" @@ -107,6 +111,21 @@ Result> OrcFileBatchReader::Create( Result> OrcFileBatchReader::GetFileSchema() const { assert(reader_); + + // If the writer stored a serialized Arrow schema with per-field metadata via + // AddMetadata, prefer that over the plain ORC type tree. + if (reader_->HasMetadataValue(ArrowUtils::kArrowSchemaMetadataKey)) { + std::string encoded = reader_->GetMetadataValue(ArrowUtils::kArrowSchemaMetadataKey); + std::string decoded = arrow::util::base64_decode(encoded); + auto buffer = arrow::Buffer::FromString(std::move(decoded)); + arrow::io::BufferReader buf_reader(buffer); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ipc::ReadSchema(&buf_reader, nullptr)); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get())); + return c_schema; + } + const auto& orc_file_type = reader_->GetOrcType(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_file_type, OrcAdapter::GetArrowType(&orc_file_type)); @@ -149,6 +168,7 @@ Status OrcFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, options_, &target_column_ids)); target_column_ids_ = target_column_ids; + previous_batch_row_count_ = 0; PAIMON_RETURN_NOT_OK(reader_->SetReadSchema(target_type, row_reader_options)); return Status::OK(); } @@ -162,7 +182,13 @@ Result>> OrcFileBatchReader::PreBuffer } Result OrcFileBatchReader::NextBatch() { - return reader_->Next(); + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->Next()); + if (BatchReader::IsEofBatch(batch)) { + previous_batch_row_count_ = 0; + } else { + previous_batch_row_count_ = batch.first->length; + } + return batch; } std::shared_ptr OrcFileBatchReader::GetReaderMetrics() const { @@ -174,56 +200,62 @@ std::shared_ptr OrcFileBatchReader::GetReaderMetrics() const { return metrics_; } -Result> OrcFileBatchReader::GetAndCheckIncludedFields( - const ::orc::Type* src_type, const ::orc::Type* target_type, - std::vector* target_column_ids) { - std::list include_fields; - std::unordered_map src_type_map; - for (uint64_t i = 0; i < src_type->getSubtypeCount(); i++) { - src_type_map[src_type->getFieldName(i)] = src_type->getSubtype(i); +Status OrcFileBatchReader::CollectTargetColumnIds(const ::orc::Type* src_type, + const ::orc::Type* target_type, + std::vector* target_column_ids) { + auto src_kind = src_type->getKind(); + auto target_kind = target_type->getKind(); + if (src_kind != target_kind) { + return Status::Invalid(fmt::format("type kind mismatch: src {} vs target {}", + src_type->toString(), target_type->toString())); } - int64_t prev_target_field_col_id = -1; - for (uint64_t i = 0; i < target_type->getSubtypeCount(); i++) { - auto& field_name = target_type->getFieldName(i); - auto iter = src_type_map.find(field_name); - if (iter == src_type_map.end()) { - return Status::Invalid( - fmt::format("field {} not in file schema {}", field_name, src_type->toString())); - } - // Noted that: do not support recall partial fields in nested type - if (iter->second->toString() != target_type->getSubtype(i)->toString()) { - return Status::Invalid( - fmt::format("target_type {} not match src_type {}, mismatch field name {}", - target_type->toString(), src_type->toString(), field_name)); + + switch (src_kind) { + case ::orc::TypeKind::STRUCT: { + std::unordered_map src_field_map; + for (uint64_t i = 0; i < src_type->getSubtypeCount(); i++) { + src_field_map[src_type->getFieldName(i)] = src_type->getSubtype(i); + } + for (uint64_t i = 0; i < target_type->getSubtypeCount(); i++) { + auto& field_name = target_type->getFieldName(i); + auto iter = src_field_map.find(field_name); + if (iter == src_field_map.end()) { + return Status::Invalid(fmt::format("field {} not in file schema {}", field_name, + src_type->toString())); + } + PAIMON_RETURN_NOT_OK(CollectTargetColumnIds( + iter->second, target_type->getSubtype(i), target_column_ids)); + } + break; } - int64_t target_field_col_id = iter->second->getColumnId(); - GetSubColumnIds(iter->second, target_column_ids); - if (prev_target_field_col_id >= target_field_col_id) { - return Status::Invalid( - "The column id of the target field should be monotonically increasing in " - "format reader"); + // Do not support partial field recall inside list/map types. + default: { + if (src_type->toString() != target_type->toString()) { + return Status::Invalid(fmt::format("type mismatch: src {} vs target {}", + src_type->toString(), target_type->toString())); + } + target_column_ids->push_back(src_type->getColumnId()); + break; } - prev_target_field_col_id = target_field_col_id; - include_fields.push_back(field_name); - } - return include_fields; -} - -void OrcFileBatchReader::GetSubColumnIds(const ::orc::Type* type, std::vector* col_ids) { - col_ids->push_back(type->getColumnId()); - for (uint64_t i = 0; i < type->getSubtypeCount(); i++) { - GetSubColumnIds(type->getSubtype(i), col_ids); } + return Status::OK(); } Result<::orc::RowReaderOptions> OrcFileBatchReader::CreateRowReaderOptions( const ::orc::Type* src_type, const ::orc::Type* target_type, std::unique_ptr<::orc::SearchArgument>&& search_arg, const std::map& options, std::vector* target_column_ids) { - PAIMON_ASSIGN_OR_RAISE(std::list include_fields, - GetAndCheckIncludedFields(src_type, target_type, target_column_ids)); + PAIMON_RETURN_NOT_OK(CollectTargetColumnIds(src_type, target_type, target_column_ids)); + for (size_t i = 1; i < target_column_ids->size(); i++) { + if ((*target_column_ids)[i - 1] >= (*target_column_ids)[i]) { + return Status::Invalid( + "The column id of the target field should be monotonically increasing in " + "format reader"); + } + } ::orc::RowReaderOptions row_reader_options; - row_reader_options.include(include_fields); + std::list include_type_ids(target_column_ids->begin(), target_column_ids->end()); + row_reader_options.includeTypes(include_type_ids); // In order to avoid issue like https://github.com/alibaba/paimon-cpp/issues/42, we explicitly // set GMT timezone. row_reader_options.setTimezoneName("GMT"); diff --git a/src/paimon/format/orc/orc_file_batch_reader.h b/src/paimon/format/orc/orc_file_batch_reader.h index 56e7a578..85673a93 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.h +++ b/src/paimon/format/orc/orc_file_batch_reader.h @@ -64,8 +64,21 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { // OrcFileBatchReader. Therefore, we need to hold BatchReader when using output ArrowArray. Result NextBatch() override; - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + uint64_t previous_first_row = reader_->GetRowNumber(); + if (previous_batch_row_count_ == 0) { + if (previous_first_row == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); + } + return previous_first_row + batch_row_id; } Result GetNumberOfRows() const override { @@ -103,18 +116,15 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { const std::shared_ptr& arrow_pool, const std::shared_ptr<::orc::MemoryPool>& orc_pool); - static void GetSubColumnIds(const ::orc::Type* type, std::vector* col_ids); - static Result<::orc::RowReaderOptions> CreateRowReaderOptions( const ::orc::Type* src_type, const ::orc::Type* target_type, std::unique_ptr<::orc::SearchArgument>&& search_arg, const std::map& options, std::vector* target_column_ids); - static Result> GetAndCheckIncludedFields( - const ::orc::Type* src_type, const ::orc::Type* target_type, - std::vector* target_column_ids); - + static Status CollectTargetColumnIds(const ::orc::Type* src_type, + const ::orc::Type* target_type, + std::vector* target_column_ids); std::map options_; std::shared_ptr arrow_pool_; @@ -125,6 +135,8 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { std::shared_ptr metrics_; std::vector target_column_ids_; std::vector> cache_ranges_; + + uint64_t previous_batch_row_count_ = 0; }; } // namespace paimon::orc diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index afb407ba..038c93e2 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -18,6 +18,7 @@ #include "paimon/format/orc/orc_file_batch_reader.h" +#include #include #include #include @@ -25,10 +26,12 @@ #include #include +#include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/ipc/api.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/defs.h" #include "paimon/format/orc/orc_adapter.h" #include "paimon/format/orc/orc_format_defs.h" @@ -45,6 +48,12 @@ namespace paimon::orc::test { +std::string SerializeSchemaToString(const std::shared_ptr& schema) { + std::shared_ptr serialized = arrow::ipc::SerializeSchema(*schema).ValueOrDie(); + return std::string(reinterpret_cast(serialized->data()), + static_cast(serialized->size())); +} + struct TestParam { uint64_t natural_read_size; bool enable_tz; @@ -156,7 +165,7 @@ class OrcFileBatchReaderTest : public ::testing::Test, const std::optional& selection_bitmap, int32_t batch_size) const { EXPECT_OK_AND_ASSIGN( auto orc_batch_reader, - OrcFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)) + OrcFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)); EXPECT_TRUE(orc_batch_reader); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); @@ -196,6 +205,50 @@ INSTANTIATE_TEST_SUITE_P(TestParam, OrcFileBatchReaderTest, ::testing::Values(TestParam{128 * 1024, false}, TestParam{16, false}, TestParam{16, true})); +TEST_F(OrcFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto file_system = dir->GetFileSystem(); + + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_schema = arrow::schema({write_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({write_field}), data_json) + .ValueOrDie()); + + std::string file_path = dir->Str() + "/" + file_name; + WriteArray(file_system, file_path, write_array, write_schema, /*options=*/{}); + + auto read_field = arrow::field("f0", arrow::binary()); + arrow::Schema read_schema({read_field}); + auto orc_batch_reader = PrepareOrcFileBatchReader(file_path, &read_schema, batch_size_, + DEFAULT_NATURAL_READ_SIZE); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + ASSERT_TRUE(file_schema->Equals(read_schema)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_field}), data_json) + .ValueOrDie()); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.orc"); + check_binary_read_result(arrow::large_binary(), "large-binary.orc"); +} + TEST_F(OrcFileBatchReaderTest, TestSetReadSchema) { std::string file_name = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/f1=10/bucket-1/" @@ -259,12 +312,13 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string orc_schema = "struct"; std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(orc_schema); std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col2", "col3"}), - row_reader_option.getIncludeNames()); + // col1(1), col2(2), col3(3) — struct container(0) is not included + ASSERT_EQ(target_column_ids, (std::vector{1, 2, 3})); ASSERT_EQ(row_reader_option.getEnableLazyDecoding(), false); } { @@ -275,28 +329,15 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string target_orc_schema = "struct"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col3"}), row_reader_option.getIncludeNames()); + // col1(1), col3(3) — struct container(0) not included, col2(2) skipped + ASSERT_EQ(target_column_ids, (std::vector{1, 3})); ASSERT_EQ(row_reader_option.getEnableLazyDecoding(), true); } - { - // read partial fields, sequence mismatch - std::map options; - std::string src_orc_schema = "struct"; - std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); - std::string target_orc_schema = "struct"; - std::unique_ptr<::orc::Type> target_type = - ::orc::Type::buildTypeFromString(target_orc_schema); - ASSERT_NOK_WITH_MSG( - OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), - /*search_arg=*/nullptr, options, - &target_column_ids), - "The column id of the target field should be monotonically increasing in format " - "reader"); - } { // read non exist column std::map options; @@ -305,12 +346,14 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string target_orc_schema = "struct"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids), "field non_exist_col not in file schema"); } { + // read partial top-level fields with nested type (all sub-fields) std::map options; std::string src_orc_schema = "struct>,sub3:int>,col2:double,col3:" @@ -320,14 +363,16 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { "struct>,sub3:int>,col3:map>"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col3"}), row_reader_option.getIncludeNames()); + // Struct IDs (0, 1) not included. Selected: sub1(2), sub2-list(3), sub3(6), col3-map(8). + ASSERT_EQ(target_column_ids, (std::vector{2, 3, 6, 8})); } { - // read with type mismatch + // read with type mismatch in nested field std::map options; std::string src_orc_schema = "struct,col2:double,col3:string>"; @@ -336,15 +381,107 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { "struct,col2:double,col3:string>"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); - + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type kind mismatch"); + } + { + // read partial sub-fields of nested struct (nested field projection) + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + // only read sub1 and sub3 from col1 + std::string target_orc_schema = "struct>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_OK_AND_ASSIGN(auto row_reader_option, + OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids)); + // src type IDs: struct(0), col1(1){sub1(2), sub2(3), sub3(4)}, col2(5) + // Struct IDs (0, 1) not included. Selected: sub1(2), sub3(4) + ASSERT_EQ(target_column_ids, (std::vector{2, 4})); + } + { + // nested struct sub-fields out-of-order should fail + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_NOK_WITH_MSG( OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids), - "target_type " - "struct,col2:double,col3:string> not match " - "src_type struct,col2:double,col3:string>, " - "mismatch field name col1"); + "The column id of the target field should be monotonically increasing in format " + "reader"); + } + { + // top-level order correct but nested sub-fields out-of-order should fail + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + // col1 before col2 (correct top-level order), but sub2 before sub1 (wrong nested order) + std::string target_orc_schema = "struct,col2:int>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG( + OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, + &target_column_ids), + "The column id of the target field should be monotonically increasing in format " + "reader"); + } + { + // decimal precision mismatch + std::map options; + std::string src_orc_schema = "struct"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); + } + { + std::map options; + std::string src_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + // list/map sub-field partial projection is not supported; toString() mismatch is reported + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); + } + { + std::map options; + std::string src_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + // list/map sub-field partial projection is not supported; toString() mismatch is reported + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); } } @@ -358,13 +495,22 @@ TEST_P(OrcFileBatchReaderTest, TestNextBatchSimple) { for (auto batch_size : {1, 2, 3, 5, 8, 10}) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, batch_size, natural_read_size); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), -1); - ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( - orc_batch_reader.get())); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 8); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); + int i = 0; + while (true) { + ASSERT_OK_AND_ASSIGN( + auto result_array, + paimon::test::ReadResultCollector::CollectResultOneBatch(orc_batch_reader.get())); + if (!result_array) { + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); + break; + } + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), i * batch_size); + ASSERT_TRUE(result_array->Equals(std::make_shared( + struct_array_->Slice(i * batch_size, result_array->length())))); + i++; + } orc_batch_reader->Close(); - auto expected_array = std::make_shared(struct_array_); - ASSERT_TRUE(result_array->Equals(expected_array)); // test metrics auto reader_metrics = orc_batch_reader->GetReaderMetrics(); ASSERT_OK_AND_ASSIGN(uint64_t io_count, @@ -632,18 +778,18 @@ TEST_F(OrcFileBatchReaderTest, TestReadNoField) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, /*batch_size=*/3, /*natural_read_size=*/10); // read 3 rows - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 0); // read 3 rows ASSERT_OK_AND_ASSIGN(auto batch2, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 3); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch3, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); // read rows with eof ASSERT_OK_AND_ASSIGN(auto batch4, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 8); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); orc_batch_reader->Close(); @@ -747,4 +893,347 @@ TEST_P(OrcFileBatchReaderTest, TestTimestampType) { // TODO(liancheng.lsz): TestBitmapPushDownWithMultiRowGroups, TestPredicateAndBitmapPushDown // TODO(liancheng.lsz): TestGenReadRanges + +TEST_F(OrcFileBatchReaderTest, TestNestedFieldProjection) { + // Write data with nested struct: struct,col2:int> + auto sub1 = arrow::field("sub1", arrow::int32()); + auto sub2 = arrow::field("sub2", arrow::float64()); + auto sub3 = arrow::field("sub3", arrow::utf8()); + auto col1 = arrow::field("col1", arrow::struct_({sub1, sub2, sub3})); + auto col2 = arrow::field("col2", arrow::int32()); + + arrow::FieldVector write_fields = {col1, col2}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[10, 1.1, "aaa"], 100], + [[20, 2.2, "bbb"], 200], + [[30, 3.3, "ccc"], 300], + [null, 400], + [[50, null, "eee"], null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/nested_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read partial sub-fields: col1.sub1 and col1.sub3 only + auto read_col1 = arrow::field("col1", arrow::struct_({sub1, sub3})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[10, "aaa"]], + [[20, "bbb"]], + [[30, "ccc"]], + [null], + [[50, "eee"]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read partial sub-fields + top-level field: col1.sub2 and col2 + auto read_col1 = arrow::field("col1", arrow::struct_({sub2})); + arrow::Schema read_schema({read_col1, col2}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1, col2}), R"([ + [[1.1], 100], + [[2.2], 200], + [[3.3], 300], + [null, 400], + [[null], null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read single nested sub-field: col1.sub1 only + auto read_col1 = arrow::field("col1", arrow::struct_({sub1})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[10]], + [[20]], + [[30]], + [null], + [[50]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestDeepNestedFieldProjection) { + // struct,e:double>,f:int> + auto field_c = arrow::field("c", arrow::int32()); + auto field_d = arrow::field("d", arrow::utf8()); + auto field_b = arrow::field("b", arrow::struct_({field_c, field_d})); + auto field_e = arrow::field("e", arrow::float64()); + auto field_a = arrow::field("a", arrow::struct_({field_b, field_e})); + auto field_f = arrow::field("f", arrow::int32()); + + arrow::FieldVector write_fields = {field_a, field_f}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[[1, "x"], 10.0], 100], + [[[2, "y"], 20.0], 200], + [null, 300] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/deep_nested_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read a.b.c only (skip a.b.d and a.e) + auto read_b = arrow::field("b", arrow::struct_({field_c})); + auto read_a = arrow::field("a", arrow::struct_({read_b})); + arrow::Schema read_schema({read_a}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_a}), R"([ + [[[1]]], + [[[2]]], + [null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestNestedFieldProjectionWithListAndMap) { + // struct,sub3:map>,col2:string> + auto sub1 = arrow::field("sub1", arrow::int32()); + auto sub2 = arrow::field("sub2", arrow::list(arrow::int32())); + auto sub3 = arrow::field("sub3", arrow::map(arrow::utf8(), arrow::int32())); + auto col1 = arrow::field("col1", arrow::struct_({sub1, sub2, sub3})); + auto col2 = arrow::field("col2", arrow::utf8()); + + arrow::FieldVector write_fields = {col1, col2}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[10, [1, 2, 3], [["a", 1], ["b", 2]]], "hello"], + [[20, [4, 5], [["c", 3]]], "world"], + [[30, null, null], null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/nested_list_map_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read col1.sub2 (list type) only — nested field projection skipping sub1 and sub3 + auto read_col1 = arrow::field("col1", arrow::struct_({sub2})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[[1, 2, 3]]], + [[[4, 5]]], + [[null]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read col1.sub1 + col1.sub3(map) — skip sub2 + auto read_col1 = arrow::field("col1", arrow::struct_({sub1, sub3})); + arrow::Schema read_schema({read_col1, col2}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1, col2}), R"([ + [[10, [["a", 1], ["b", 2]]], "hello"], + [[20, [["c", 3]]], "world"], + [[30, null], null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestListStructPartialProjection) { + // Verify that array> read as array> is rejected. + // Partial projection inside list/map elements is not supported; the list type toString() + // comparison catches the mismatch and returns an error before any ORC batch is read. + auto field_a = arrow::field("a", arrow::int32()); + auto field_b = arrow::field("b", arrow::float64()); + auto col1 = arrow::field("col1", arrow::list(arrow::struct_({field_a, field_b}))); + + arrow::FieldVector write_fields = {col1}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[[1, 1.1], [2, 2.2]]], + [[[3, 3.3]]], + [[null]], + [null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/list_struct_projection_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + auto read_col1 = arrow::field("col1", arrow::list(arrow::struct_({field_a}))); + arrow::Schema read_schema({read_col1}); + std::shared_ptr file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(data_path)); + ASSERT_OK_AND_ASSIGN(auto in_stream, + OrcInputStreamImpl::Create(input_stream, DEFAULT_NATURAL_READ_SIZE)); + ASSERT_OK_AND_ASSIGN( + auto orc_batch_reader, + OrcFileBatchReader::Create(std::move(in_stream), pool_, /*options=*/{}, /*batch_size=*/10)); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(read_schema, c_schema.get()).ok()); + ASSERT_NOK_WITH_MSG(orc_batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "type mismatch"); +} + +TEST_F(OrcFileBatchReaderTest, TestAddMetadataPerFieldMetadata) { + // Write a simple ORC file, call AddMetadata to inject per-field metadata + // before Finish, then read back and verify the file schema carries the metadata. + auto write_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::float64()), + }); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + std::string file_path = dir->Str() + "/update_schema_test.orc"; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(auto orc_output_stream, OrcOutputStreamImpl::Create(out)); + ASSERT_OK_AND_ASSIGN(auto format_writer, + OrcFormatWriter::Create(std::move(orc_output_stream), *write_schema, + /*options=*/{}, "zstd", + /*batch_size=*/10, pool_)); + + // Write one batch of data. + auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(write_schema->fields()), + R"([[1, "alice", 95.5], [2, "bob", 88.0], [3, "charlie", 72.3]])") + .ValueOrDie(); + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*data, &c_array).ok()); + ASSERT_OK(format_writer->AddBatch(&c_array)); + ASSERT_OK(format_writer->Flush()); + + // Build an updated schema with per-field metadata on "name" and "score". + auto name_meta = std::make_shared(); + name_meta->Append("shredding.field_mapping", "0:alice,1:bob,2:charlie"); + name_meta->Append("shredding.num_columns", "3"); + auto score_meta = std::make_shared(); + score_meta->Append("custom.unit", "percent"); + + auto updated_schema = arrow::schema({ + write_schema->field(0), // id — no metadata + write_schema->field(1)->WithMetadata(name_meta), // name — shredding metadata + write_schema->field(2)->WithMetadata(score_meta), // score — custom metadata + }); + + // AddMetadata must be called before Finish. + ASSERT_OK(format_writer->AddMetadata( + {{ArrowUtils::kArrowSchemaMetadataKey, SerializeSchemaToString(updated_schema)}})); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + + // Read back: GetFileSchema should reflect the updated per-field metadata. + auto orc_batch_reader = PrepareOrcFileBatchReader(file_path, write_schema.get(), batch_size_, + DEFAULT_NATURAL_READ_SIZE); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Field 0 "id": no metadata. + ASSERT_EQ("id", file_schema->field(0)->name()); + + // Field 1 "name": should have the shredding metadata we set. + ASSERT_EQ("name", file_schema->field(1)->name()); + auto read_name_meta = file_schema->field(1)->metadata(); + ASSERT_NE(nullptr, read_name_meta); + auto field_mapping_val = read_name_meta->Get("shredding.field_mapping").ValueOrDie(); + ASSERT_EQ("0:alice,1:bob,2:charlie", field_mapping_val); + auto num_columns_val = read_name_meta->Get("shredding.num_columns").ValueOrDie(); + ASSERT_EQ("3", num_columns_val); + + // Field 2 "score": should have the custom metadata. + ASSERT_EQ("score", file_schema->field(2)->name()); + auto read_score_meta = file_schema->field(2)->metadata(); + ASSERT_NE(nullptr, read_score_meta); + auto unit_val = read_score_meta->Get("custom.unit").ValueOrDie(); + ASSERT_EQ("percent", unit_val); + + // Also verify data integrity — read it back and compare content. + ASSERT_OK_AND_ASSIGN(auto result_array, + paimon::test::ReadResultCollector::CollectResult(orc_batch_reader.get())); + ASSERT_EQ(result_array->num_chunks(), 1); + ASSERT_TRUE(data->Equals(*result_array->chunk(0))) << result_array->ToString(); +} + } // namespace paimon::orc::test diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp index 870d755d..1a394ca9 100644 --- a/src/paimon/format/orc/orc_format_writer.cpp +++ b/src/paimon/format/orc/orc_format_writer.cpp @@ -22,19 +22,25 @@ #include #include #include +#include #include +#include +#include #include #include "arrow/api.h" #include "arrow/array/array_base.h" #include "arrow/c/bridge.h" +#include "arrow/util/base64.h" #include "fmt/format.h" #include "orc/Common.hh" #include "orc/OrcFile.hh" #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/options/memory_size.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -75,6 +81,9 @@ Result> OrcFormatWriter::Create( const std::map& options, const std::string& compression, int32_t batch_size, const std::shared_ptr& pool) { assert(output_stream); + if (VariantTypeUtils::ContainsVariantField(arrow::schema(schema.fields()))) { + return Status::NotImplemented("ORC format does not support the VARIANT type"); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::orc::Type> orc_type, OrcAdapter::GetOrcType(schema)); auto data_type = arrow::struct_(schema.fields()); try { @@ -207,6 +216,38 @@ std::shared_ptr OrcFormatWriter::GetWriterMetrics() const { return metrics_; } +Status OrcFormatWriter::AddMetadata(const std::map& metadata) { + if (metadata.empty()) { + return Status::OK(); + } + try { + for (const auto& [key, value] : metadata) { + writer_->addUserMetadata(key, arrow::util::base64_encode(std::string_view(value))); + } + } catch (const std::exception& e) { + return Status::Invalid( + fmt::format("orc format writer AddMetadata failed for file {}, with {} error", + output_stream_->getName(), e.what())); + } catch (...) { + return Status::UnknownError( + fmt::format("orc format writer AddMetadata failed for file {}, with unknown error", + output_stream_->getName())); + } + return Status::OK(); +} + +namespace { + +Result GetMemorySizeOption(const std::map& options, + const std::string& key, uint64_t default_value) { + PAIMON_ASSIGN_OR_RAISE(std::string value, OptionsUtils::GetValueFromMap( + options, key, std::to_string(default_value))); + PAIMON_ASSIGN_OR_RAISE(int64_t bytes, MemorySize::ParseBytes(value)); + return static_cast(bytes); +} + +} // namespace + Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( const std::map& options, const std::string& file_compression, const std::shared_ptr& data_type) { @@ -220,15 +261,15 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( } } ::orc::WriterOptions writer_options; - PAIMON_ASSIGN_OR_RAISE(size_t stripe_size, OptionsUtils::GetValueFromMap( - options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE)); + PAIMON_ASSIGN_OR_RAISE(uint64_t stripe_size, + GetMemorySizeOption(options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE)); writer_options.setStripeSize(stripe_size); PAIMON_ASSIGN_OR_RAISE(::orc::CompressionKind compression, ToOrcCompressionKind(StringUtils::ToLowerCase(file_compression))); writer_options.setCompression(compression); - PAIMON_ASSIGN_OR_RAISE(size_t compression_block_size, OptionsUtils::GetValueFromMap( - options, ORC_COMPRESSION_BLOCK_SIZE, - DEFAULT_COMPRESSION_BLOCK_SIZE)); + PAIMON_ASSIGN_OR_RAISE( + uint64_t compression_block_size, + GetMemorySizeOption(options, ORC_COMPRESSION_BLOCK_SIZE, DEFAULT_COMPRESSION_BLOCK_SIZE)); writer_options.setCompressionBlockSize(compression_block_size); PAIMON_ASSIGN_OR_RAISE( double dictionary_key_threshold, @@ -237,9 +278,9 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( writer_options.setDictionaryKeySizeThreshold(dictionary_key_threshold); // always use tight numeric vector writer_options.setUseTightNumericVector(true); - PAIMON_ASSIGN_OR_RAISE(size_t row_index_stride, - OptionsUtils::GetValueFromMap(options, ORC_ROW_INDEX_STRIDE, - DEFAULT_ROW_INDEX_STRIDE)); + PAIMON_ASSIGN_OR_RAISE(uint64_t row_index_stride, + OptionsUtils::GetValueFromMap(options, ORC_ROW_INDEX_STRIDE, + DEFAULT_ROW_INDEX_STRIDE)); writer_options.setRowIndexStride(row_index_stride); // In order to avoid issue like https://github.com/alibaba/paimon-cpp/issues/42, we explicitly // set GMT timezone. diff --git a/src/paimon/format/orc/orc_format_writer.h b/src/paimon/format/orc/orc_format_writer.h index 409e7a87..1a646424 100644 --- a/src/paimon/format/orc/orc_format_writer.h +++ b/src/paimon/format/orc/orc_format_writer.h @@ -66,6 +66,8 @@ class OrcFormatWriter : public FormatWriter { std::shared_ptr GetWriterMetrics() const override; + Status AddMetadata(const std::map& metadata) override; + private: OrcFormatWriter(const std::shared_ptr& orc_memory_pool, std::unique_ptr<::orc::OutputStream>&& output_stream, diff --git a/src/paimon/format/orc/orc_format_writer_test.cpp b/src/paimon/format/orc/orc_format_writer_test.cpp index 5026e957..0f531f52 100644 --- a/src/paimon/format/orc/orc_format_writer_test.cpp +++ b/src/paimon/format/orc/orc_format_writer_test.cpp @@ -29,6 +29,8 @@ #include "arrow/array/builder_primitive.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/ipc/api.h" #include "gtest/gtest.h" #include "orc/Common.hh" #include "orc/MemoryPool.hh" @@ -37,6 +39,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/format/orc/orc_format_defs.h" #include "paimon/format/orc/orc_input_stream_impl.h" #include "paimon/format/orc/orc_metrics.h" @@ -295,6 +298,23 @@ TEST_F(OrcFormatWriterTest, TestPrepareWriterOptions) { "invalid config, do not support writing timestamp with timezone in legacy format"); } } +TEST_F(OrcFormatWriterTest, TestVariantNotSupported) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + std::string file_name = test_root + "/variant.orc"; + auto arrow_schema = + arrow::schema({arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + file_system_->Create(file_name, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output_stream, + OrcOutputStreamImpl::Create(out)); + auto result = OrcFormatWriter::Create(std::move(output_stream), *arrow_schema, options, + /*compression=*/"lz4", /*batch_size=*/16, pool_); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); +} + // TODO(liancheng.lsz): add tests for GetEstimateLength } // namespace paimon::orc::test diff --git a/src/paimon/format/orc/orc_input_output_stream_test.cpp b/src/paimon/format/orc/orc_input_output_stream_test.cpp index 1caa5f10..7db0e1bb 100644 --- a/src/paimon/format/orc/orc_input_output_stream_test.cpp +++ b/src/paimon/format/orc/orc_input_output_stream_test.cpp @@ -32,6 +32,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/utils/path_util.h" #include "paimon/format/orc/orc_format_defs.h" #include "paimon/format/orc/orc_input_stream_impl.h" #include "paimon/format/orc/orc_output_stream_impl.h" @@ -55,7 +56,8 @@ TEST(OrcInputOutputStreamTest, TestInOutStream) { file_system->Create(file_name, /*overwrite=*/true)); ASSERT_OK_AND_ASSIGN(std::unique_ptr out_stream, OrcOutputStreamImpl::Create(out)); - ASSERT_EQ(out_stream->getName(), file_name); + ASSERT_OK_AND_ASSIGN(auto normalized_file_name, PathUtil::NormalizePath(file_name)); + ASSERT_EQ(out_stream->getName(), normalized_file_name); ASSERT_EQ(out_stream->getNaturalWriteSize(), 128 * 1024); ASSERT_EQ(out_stream->getLength(), 0); @@ -86,7 +88,8 @@ TEST(OrcInputOutputStreamTest, TestSimple) { file_system->Create(file_name, /*overwrite=*/true)); ASSERT_OK_AND_ASSIGN(std::unique_ptr out_stream, OrcOutputStreamImpl::Create(out)); - ASSERT_EQ(out_stream->getName(), file_name); + ASSERT_OK_AND_ASSIGN(auto normalized_file_name, PathUtil::NormalizePath(file_name)); + ASSERT_EQ(out_stream->getName(), normalized_file_name); ASSERT_EQ(out_stream->getNaturalWriteSize(), 128 * 1024); ASSERT_EQ(out_stream->getLength(), 0); @@ -133,7 +136,7 @@ TEST(OrcInputOutputStreamTest, TestSimple) { ASSERT_OK_AND_ASSIGN(auto in_stream, OrcInputStreamImpl::Create(input_stream, DEFAULT_NATURAL_READ_SIZE)); auto length = file_system->GetFileStatus(file_name).value()->GetLen(); - ASSERT_EQ(in_stream->getName(), file_name); + ASSERT_EQ(in_stream->getName(), normalized_file_name); ASSERT_EQ(in_stream->getLength(), length); ASSERT_EQ(in_stream->getNaturalReadSize(), 1024 * 1024); diff --git a/src/paimon/format/orc/orc_input_stream_impl.cpp b/src/paimon/format/orc/orc_input_stream_impl.cpp index bc5dec27..572ae814 100644 --- a/src/paimon/format/orc/orc_input_stream_impl.cpp +++ b/src/paimon/format/orc/orc_input_stream_impl.cpp @@ -24,6 +24,7 @@ #include "fmt/format.h" #include "orc/Exceptions.hh" #include "orc/Reader.hh" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/status.h" @@ -31,9 +32,10 @@ namespace paimon::orc { Result> OrcInputStreamImpl::Create( const std::shared_ptr& input_stream, uint64_t natural_read_size) { PAIMON_ASSIGN_OR_RAISE(std::string name, input_stream->GetUri()); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream->Length()); - return std::unique_ptr( - new OrcInputStreamImpl(input_stream, name, length, natural_read_size)); + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "file length")); + return std::unique_ptr(new OrcInputStreamImpl( + input_stream, name, static_cast(length), natural_read_size)); } OrcInputStreamImpl::OrcInputStreamImpl(const std::shared_ptr& input_stream, @@ -63,11 +65,20 @@ void OrcInputStreamImpl::read(void* buf, uint64_t length, uint64_t offset) { metrics_->IOCount.fetch_add(1); } - Result read_bytes = input_stream_->Read(static_cast(buf), length, offset); + Status status = ValidateValueInRange(length, "read length"); + if (!status.ok()) { + throw ::orc::ParseError(status.ToString()); + } + status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + throw ::orc::ParseError(status.ToString()); + } + Result read_bytes = input_stream_->Read( + static_cast(buf), static_cast(length), static_cast(offset)); if (!read_bytes.ok()) { throw ::orc::ParseError("read failed, status: " + read_bytes.status().ToString()); } - if (static_cast(read_bytes.value()) != length) { + if (read_bytes.value() != static_cast(length)) { throw ::orc::ParseError( fmt::format("read failed, expected length: {}, actual read length: {}", length, read_bytes.value())); diff --git a/src/paimon/format/orc/orc_output_stream_impl.cpp b/src/paimon/format/orc/orc_output_stream_impl.cpp index 1168e457..abe92edb 100644 --- a/src/paimon/format/orc/orc_output_stream_impl.cpp +++ b/src/paimon/format/orc/orc_output_stream_impl.cpp @@ -23,6 +23,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/status.h" @@ -44,17 +45,26 @@ uint64_t OrcOutputStreamImpl::getLength() const { if (!pos.ok()) { throw std::runtime_error(fmt::format("get length failed, file name {}, error msg {}", file_name_, pos.status().ToString())); - } else { - return pos.value(); } + Status status = ValidateValueInRange(pos.value(), "file position"); + if (!status.ok()) { + throw std::runtime_error(fmt::format("get length failed, file name {}, error msg {}", + file_name_, status.ToString())); + } + return static_cast(pos.value()); } void OrcOutputStreamImpl::write(const void* buf, size_t length) { - Result write_len = output_stream_->Write(static_cast(buf), length); + Status status = ValidateValueInRange(length, "write length"); + if (!status.ok()) { + throw std::runtime_error("write failed, status: " + status.ToString()); + } + Result write_len = + output_stream_->Write(static_cast(buf), static_cast(length)); if (!write_len.ok()) { throw std::runtime_error("write failed, status: " + write_len.status().ToString()); } - if (static_cast(write_len.value()) != length) { + if (write_len.value() != static_cast(length)) { throw std::runtime_error( fmt::format("write failed, expected length: {}, actual write length: {}", length, write_len.value())); diff --git a/src/paimon/format/orc/orc_reader_wrapper.cpp b/src/paimon/format/orc/orc_reader_wrapper.cpp index 20956063..f1482690 100644 --- a/src/paimon/format/orc/orc_reader_wrapper.cpp +++ b/src/paimon/format/orc/orc_reader_wrapper.cpp @@ -51,6 +51,7 @@ Status OrcReaderWrapper::SetReadSchema(const std::shared_ptr& t try { row_reader_ = reader_->createRowReader(row_reader_options); target_type_ = target_type; + next_row_ = 0; } catch (const std::exception& e) { return Status::Invalid( fmt::format("orc file batch reader create row reader failed for file {}, with {} error", diff --git a/src/paimon/format/orc/orc_reader_wrapper.h b/src/paimon/format/orc/orc_reader_wrapper.h index e98ad3de..f01ccd5a 100644 --- a/src/paimon/format/orc/orc_reader_wrapper.h +++ b/src/paimon/format/orc/orc_reader_wrapper.h @@ -87,6 +87,14 @@ class OrcReaderWrapper { return reader_->getType(); } + bool HasMetadataValue(const std::string& key) const { + return reader_->hasMetadataValue(key); + } + + std::string GetMetadataValue(const std::string& key) const { + return reader_->getMetadataValue(key); + } + Result>> GenReadRanges( std::vector target_column_ids, uint64_t begin_row_num, uint64_t end_row_num, bool* need_prefetch) const { diff --git a/src/paimon/format/orc/predicate_converter_test.cpp b/src/paimon/format/orc/predicate_converter_test.cpp index c5726e5b..f9ec5557 100644 --- a/src/paimon/format/orc/predicate_converter_test.cpp +++ b/src/paimon/format/orc/predicate_converter_test.cpp @@ -33,6 +33,16 @@ namespace paimon::orc::test { +namespace { + +// Use an explicit int64_t literal for BIGINT predicates. On macOS arm64, `long` and +// `int64_t` are distinct types, so `Literal(5l)` may instantiate `Literal`. +Literal BigIntLiteral(int64_t value) { + return Literal(value); +} + +} // namespace + TEST(PredicateConverterTest, TestSimple) { std::string orc_schema = "struct"; @@ -51,7 +61,7 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) = 5), expr = leaf-0", search_arg->toString()); } @@ -72,19 +82,19 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) = 5), expr = (not leaf-0)", search_arg->toString()); } { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) <= 5), expr = (not leaf-0)", search_arg->toString()); } { auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) < 5), expr = (not leaf-0)", search_arg->toString()); } @@ -97,20 +107,20 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) < 5), expr = leaf-0", search_arg->toString()); } { auto predicate = PredicateBuilder::LessOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) <= 5), expr = leaf-0", search_arg->toString()); } { - auto predicate = - PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l), Literal(5l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(3), BigIntLiteral(5)}); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) in [1, 3, 5]), expr = leaf-0", search_arg->toString()); } @@ -147,9 +157,9 @@ TEST(PredicateConverterTest, TestSimple) { ASSERT_EQ("expr = YES", search_arg->toString()); } { - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l), Literal(5l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(3), BigIntLiteral(5)}); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) in [1, 3, 5]), expr = (not leaf-0)", search_arg->toString()); @@ -189,7 +199,7 @@ TEST(PredicateConverterTest, TestCompound) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + BigIntLiteral(3)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -223,7 +233,7 @@ TEST(PredicateConverterTest, TestCompound) { auto predicate, PredicateBuilder::Or({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + BigIntLiteral(3)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -247,7 +257,7 @@ TEST(PredicateConverterTest, TestCompound) { {PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, Literal(true)), PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l))}) + FieldType::BIGINT, BigIntLiteral(3))}) .value(), PredicateBuilder::And( {PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", diff --git a/src/paimon/format/orc/predicate_pushdown_test.cpp b/src/paimon/format/orc/predicate_pushdown_test.cpp index 643d435f..76d1e740 100644 --- a/src/paimon/format/orc/predicate_pushdown_test.cpp +++ b/src/paimon/format/orc/predicate_pushdown_test.cpp @@ -55,6 +55,16 @@ class Predicate; namespace paimon::orc::test { +namespace { + +// Use an explicit int64_t literal for BIGINT predicates. On macOS arm64, `long` and +// `int64_t` are distinct types, so `Literal(5l)` may instantiate `Literal`. +Literal BigIntLiteral(int64_t value) { + return Literal(value); +} + +} // namespace + class PredicatePushdownTest : public ::testing::Test { public: void SetUp() override { @@ -175,48 +185,48 @@ TEST_F(PredicatePushdownTest, TestIntDoubleData) { { // f2 != 4, has data auto predicate = PredicateBuilder::NotEqual(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(4l)); + FieldType::BIGINT, BigIntLiteral(4)); CheckResult(read_schema, predicate, expected_array); } { // f2 == 6, has data auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)); + FieldType::BIGINT, BigIntLiteral(6)); CheckResult(read_schema, predicate, expected_array); } { // f2 == 1, no data auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(1l)); + FieldType::BIGINT, BigIntLiteral(1)); CheckResult(read_schema, predicate, /*expected_array=*/ nullptr); } { // f2 in [1,2,3], no data - auto predicate = - PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(2l), Literal(3l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2), BigIntLiteral(3)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f2 not in [1,2,3], has data - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(2l), Literal(3l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2), BigIntLiteral(3)}); CheckResult(read_schema, predicate, expected_array); } { // f2 in [2,3,4], has data - auto predicate = - PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(2l), Literal(3l), Literal(4l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(2), BigIntLiteral(3), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } { // f2 not in [2,3,4], has data - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(2l), Literal(3l), Literal(4l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(2), BigIntLiteral(3), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } } @@ -363,26 +373,28 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithAllDataNull) { // other predicate, always return IS_NULL (no data) { // f4 in [1,2], no data - auto predicate = PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, {Literal(1l), Literal(2l)}); + auto predicate = + PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"f4", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 not in [1,2], no data - auto predicate = PredicateBuilder::NotIn(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, {Literal(1l), Literal(2l)}); + auto predicate = + PredicateBuilder::NotIn(/*field_index=*/4, /*field_name=*/"f4", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 >= 3, no data auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, BigIntLiteral(3)); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 <= 3, no data auto predicate = PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, BigIntLiteral(3)); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } } @@ -436,14 +448,14 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithNullLiteral) { // f2 in [1,null,2], no data auto predicate = PredicateBuilder::In( /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(2l)}); + {BigIntLiteral(1), Literal(FieldType::BIGINT), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f2 in [1,null,2,4], has data auto predicate = PredicateBuilder::In( /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(2l), Literal(4l)}); + {BigIntLiteral(1), Literal(FieldType::BIGINT), BigIntLiteral(2), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } } @@ -457,7 +469,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, @@ -471,7 +483,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f3", @@ -485,7 +497,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", @@ -499,7 +511,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", @@ -513,7 +525,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0)))})); ASSERT_TRUE(predicate); @@ -525,7 +537,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0)))})); ASSERT_TRUE(predicate); @@ -536,7 +548,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { ASSERT_OK_AND_ASSIGN( auto predicate, PredicateBuilder::Or({PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", FieldType::BINARY)})); ASSERT_TRUE(predicate); @@ -548,7 +560,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, @@ -562,7 +574,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f3", diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 3dedd91c..a1a566c0 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -18,13 +18,16 @@ set(PAIMON_PARQUET_FILE_FORMAT parquet_field_id_converter.cpp predicate_converter.cpp file_reader_wrapper.cpp + page_filtered_row_group_reader.cpp parquet_timestamp_converter.cpp parquet_file_batch_reader.cpp parquet_file_format_factory.cpp parquet_format_writer.cpp parquet_schema_util.cpp parquet_stats_extractor.cpp - parquet_writer_builder.cpp) + parquet_writer_builder.cpp + row_ranges.cpp + column_index_filter.cpp) add_paimon_lib(paimon_parquet_file_format SOURCES @@ -37,17 +40,20 @@ add_paimon_lib(paimon_parquet_file_format arrow glog fmt - dl Threads::Threads SHARED_LINK_LIBS paimon_shared SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) +target_include_directories(paimon_parquet_file_format_objlib SYSTEM + PRIVATE "${ARROW_SOURCE_DIR}/cpp/src") + if(PAIMON_BUILD_TESTS) add_paimon_test(parquet_format_test SOURCES file_reader_wrapper_test.cpp + page_filtered_row_group_reader_test.cpp parquet_timestamp_converter_test.cpp parquet_field_id_converter_test.cpp parquet_file_batch_reader_test.cpp @@ -56,13 +62,13 @@ if(PAIMON_BUILD_TESTS) parquet_writer_builder_test.cpp predicate_converter_test.cpp predicate_pushdown_test.cpp + column_index_filter_test.cpp + variant_parquet_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_parquet_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} parquet ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp new file mode 100644 index 00000000..61849802 --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -0,0 +1,702 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/column_index_filter.h" + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/data/decimal.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/literal.h" + +namespace paimon::parquet { + +Result ColumnIndexFilter::CalculateRowRanges( + const std::shared_ptr& predicate, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + const std::map& column_name_to_index, int32_t row_group_index, + int64_t row_group_row_count) { + if (!predicate || !page_index_reader) { + return RowRanges::CreateSingle(row_group_row_count); + } + + auto rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + if (!rg_page_index_reader) { + return RowRanges::CreateSingle(row_group_row_count); + } + + return VisitPredicate(predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader.get()); +} + +Result ColumnIndexFilter::VisitPredicate( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { + return VisitLeafPredicate(leaf_predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader); + } + + if (auto compound_predicate = std::dynamic_pointer_cast(predicate)) { + return VisitCompoundPredicate(compound_predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader); + } + + return Status::Invalid("Unknown predicate type"); +} + +Result ColumnIndexFilter::VisitLeafPredicate( + const std::shared_ptr& leaf_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + const std::string& field_name = leaf_predicate->FieldName(); + auto it = column_name_to_index.find(field_name); + if (it == column_name_to_index.end()) { + // Predicates referencing fields absent from the data file are stripped + // upstream by FieldMappingBuilder, so reaching here indicates a contract + // violation by the caller. + return Status::Invalid( + fmt::format("column '{}' not found in column_name_to_index", field_name)); + } + const auto& function = leaf_predicate->GetFunction(); + auto function_type = function.GetType(); + + int32_t column_index = it->second; + auto column_index_ptr = rg_page_index_reader->GetColumnIndex(column_index); + auto offset_index_ptr = rg_page_index_reader->GetOffsetIndex(column_index); + + if (!column_index_ptr || !offset_index_ptr) { + // Column index or offset index not available, return all rows + return RowRanges::CreateSingle(row_group_row_count); + } + + const auto& literals = leaf_predicate->Literals(); + FieldType field_type = leaf_predicate->GetFieldType(); + + if (function_type != Function::Type::IS_NULL && function_type != Function::Type::IS_NOT_NULL && + literals.empty()) { + return Status::Invalid( + fmt::format("predicate on column '{}' requires at least one literal", field_name)); + } + std::vector matching_pages; + + switch (function_type) { + case Function::Type::IS_NULL: + matching_pages = FilterPagesByIsNull(column_index_ptr); + break; + case Function::Type::IS_NOT_NULL: + matching_pages = FilterPagesByIsNotNull(column_index_ptr); + break; + case Function::Type::EQUAL: + matching_pages = FilterPagesByEqual(column_index_ptr, literals[0], field_type); + break; + case Function::Type::NOT_EQUAL: + matching_pages = FilterPagesByNotEqual(column_index_ptr, literals[0], field_type); + break; + case Function::Type::LESS_THAN: + matching_pages = FilterPagesByLessThan(column_index_ptr, literals[0], field_type); + break; + case Function::Type::LESS_OR_EQUAL: + matching_pages = FilterPagesByLessOrEqual(column_index_ptr, literals[0], field_type); + break; + case Function::Type::GREATER_THAN: + matching_pages = FilterPagesByGreaterThan(column_index_ptr, literals[0], field_type); + break; + case Function::Type::GREATER_OR_EQUAL: + matching_pages = FilterPagesByGreaterOrEqual(column_index_ptr, literals[0], field_type); + break; + case Function::Type::IN: + matching_pages = FilterPagesByIn(column_index_ptr, literals, field_type); + break; + case Function::Type::NOT_IN: + matching_pages = FilterPagesByNotIn(column_index_ptr, literals); + break; + default: + // Unsupported function type for column index filtering + return RowRanges::CreateSingle(row_group_row_count); + } + + return BuildRowRangesFromPageIndices(matching_pages, offset_index_ptr, row_group_row_count); +} + +Result ColumnIndexFilter::VisitCompoundPredicate( + const std::shared_ptr& compound_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + const auto& children = compound_predicate->Children(); + const auto& function = compound_predicate->GetFunction(); + auto function_type = function.GetType(); + + if (children.empty()) { + return RowRanges::CreateSingle(row_group_row_count); + } + + // Calculate row ranges for first child + PAIMON_ASSIGN_OR_RAISE(RowRanges result, + VisitPredicate(children[0], column_name_to_index, row_group_row_count, + rg_page_index_reader)); + + if (function_type == Function::Type::AND) { + // Short-circuit: if result is empty, no need to continue + if (result.IsEmpty()) { + return result; + } + + for (size_t i = 1; i < children.size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(RowRanges child_ranges, + VisitPredicate(children[i], column_name_to_index, + row_group_row_count, rg_page_index_reader)); + + result = RowRanges::Intersection(result, child_ranges); + + // Short-circuit: if result is empty, no need to continue + if (result.IsEmpty()) { + return result; + } + } + } else if (function_type == Function::Type::OR) { + // Short-circuit: if result already covers all rows, no need to continue + if (result.RowCount() == row_group_row_count) { + return result; + } + + for (size_t i = 1; i < children.size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(RowRanges child_ranges, + VisitPredicate(children[i], column_name_to_index, + row_group_row_count, rg_page_index_reader)); + + result = RowRanges::Union(result, child_ranges); + + // Short-circuit: if result already covers all rows, no need to continue + if (result.RowCount() == row_group_row_count) { + return result; + } + } + } else { + return Status::Invalid("Unknown compound predicate type"); + } + + return result; +} + +std::vector ColumnIndexFilter::FilterPagesByEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + + if (literal.IsNull()) { + // value = NULL is UNKNOWN for any value. No rows can match. + return matching_pages; + } + + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainEqual(min_values[i], max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByNotEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + + if (literal.IsNull()) { + // value != NULL is UNKNOWN for any value. No rows can match. + return matching_pages; + } + + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // Null-only pages: NULL != x is NULL (UNKNOWN) in SQL semantics, + // which evaluates to false. Skip null-only pages for NOT_EQUAL. + continue; + } + + // Try to exclude pages where min == max == literal (all non-null values equal literal). + // NULL != literal is NULL (UNKNOWN) in SQL, so nulls don't produce true either. + auto cmp_min = CompareEncodedWithLiteral(min_values[i], literal, field_type); + auto cmp_max = CompareEncodedWithLiteral(max_values[i], literal, field_type); + if (cmp_min.has_value() && cmp_max.has_value() && *cmp_min == 0 && *cmp_max == 0) { + // min == max == literal: all non-null values equal literal, and nulls + // don't satisfy != either. Skip this page entirely. + continue; + } + + matching_pages.push_back(i); + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByLessThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainLessThan(min_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByLessOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainLessOrEqual(min_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByGreaterThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainGreaterThan(max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByGreaterOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainGreaterOrEqual(max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIsNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& null_counts = column_index->null_counts(); + bool has_null_counts = column_index->has_null_counts(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + matching_pages.push_back(i); + continue; + } + + if (has_null_counts && null_counts[i] > 0) { + matching_pages.push_back(i); + } else if (!has_null_counts) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIsNotNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (!null_pages[i]) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals, FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + const auto& null_counts = column_index->null_counts(); + bool has_null_counts = column_index->has_null_counts(); + auto num_pages = static_cast(null_pages.size()); + + bool has_null = + std::any_of(literals.begin(), literals.end(), [](const Literal& l) { return l.IsNull(); }); + + // Pages outer loop, literals inner loop with early break when page is matched. + // Naturally produces sorted output, avoids unordered_set overhead. + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // All-null page: include only if IN list contains null + if (has_null) { + matching_pages.push_back(i); + } + continue; + } + + // Check null-in-list match for non-all-null pages + if (has_null) { + if ((has_null_counts && null_counts[i] > 0) || !has_null_counts) { + matching_pages.push_back(i); + continue; // Already matched, skip literal checks + } + } + + // Check non-null literals against page min/max with early break + for (const auto& literal : literals) { + if (literal.IsNull()) { + continue; + } + if (PageMightContainEqual(min_values[i], max_values[i], literal, field_type)) { + matching_pages.push_back(i); + break; // Page matched, no need to check more literals + } + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByNotIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + auto num_pages = static_cast(null_pages.size()); + + bool has_null = false; + for (const auto& literal : literals) { + if (literal.IsNull()) { + has_null = true; + break; + } + } + + if (has_null) { + // NOT_IN list contains null → value NOT IN (..., NULL, ...) evaluates to + // UNKNOWN for every value (because it expands to AND(..., value != NULL, ...) + // and value != NULL is always UNKNOWN). No rows can match. + return matching_pages; + } + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // Null-only pages: NULL NOT IN (non-null values) is UNKNOWN, skip. + continue; + } + + // Non-null pages could contain values not in the list + matching_pages.push_back(i); + } + + return matching_pages; +} + +RowRanges ColumnIndexFilter::BuildRowRangesFromPageIndices( + const std::vector& page_indices, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count) { + if (page_indices.empty()) { + return RowRanges::CreateEmpty(); + } + + const auto& page_locations = offset_index->page_locations(); + RowRanges ranges; + + for (int32_t page_idx : page_indices) { + if (page_idx < 0 || page_idx >= static_cast(page_locations.size())) { + continue; + } + + int64_t first_row_index = page_locations[page_idx].first_row_index; + + int64_t last_row_index; + if (page_idx + 1 < static_cast(page_locations.size())) { + last_row_index = page_locations[page_idx + 1].first_row_index - 1; + } else { + last_row_index = row_group_row_count - 1; + } + + ranges.Add(RowRanges::Range(first_row_index, last_row_index)); + } + + return ranges; +} + +std::optional ColumnIndexFilter::CompareEncodedWithLiteral(const std::string& encoded, + const Literal& literal, + FieldType field_type) { + if (literal.IsNull()) { + return std::nullopt; + } + + switch (field_type) { + case FieldType::BOOLEAN: { + if (encoded.size() < 1) { + return std::nullopt; + } + int32_t enc_val = (encoded[0] != 0) ? 1 : 0; + int32_t lit_val = literal.GetValue() ? 1 : 0; + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::DATE: { + if (encoded.size() < sizeof(int32_t)) { + return std::nullopt; + } + int32_t enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(int32_t)); + int32_t lit_val; + if (field_type == FieldType::TINYINT) { + lit_val = static_cast(literal.GetValue()); + } else if (field_type == FieldType::SMALLINT) { + lit_val = static_cast(literal.GetValue()); + } else { + lit_val = literal.GetValue(); + } + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::BIGINT: { + if (encoded.size() < sizeof(int64_t)) { + return std::nullopt; + } + int64_t enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(int64_t)); + auto lit_val = literal.GetValue(); + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::FLOAT: { + if (encoded.size() < sizeof(float)) { + return std::nullopt; + } + float enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(float)); + auto lit_val = literal.GetValue(); + return FieldsComparator::CompareFloatingPoint(enc_val, lit_val); + } + case FieldType::DOUBLE: { + if (encoded.size() < sizeof(double)) { + return std::nullopt; + } + double enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(double)); + auto lit_val = literal.GetValue(); + return FieldsComparator::CompareFloatingPoint(enc_val, lit_val); + } + case FieldType::STRING: + case FieldType::BINARY: { + auto lit_val = literal.GetValue(); + int cmp = encoded.compare(lit_val); + return (cmp < 0) ? -1 : (cmp > 0) ? 1 : 0; + } + case FieldType::DECIMAL: { + // Parquet stores DECIMAL as INT32, INT64, or FIXED_LEN_BYTE_ARRAY depending + // on precision. All are stored as unscaled integer values. + auto lit_decimal = literal.GetValue(); + Decimal::int128_t lit_val = lit_decimal.Value(); + Decimal::int128_t enc_val; + + if (encoded.size() == sizeof(int32_t)) { + // INT32 physical type (precision <= 9) + int32_t raw; + std::memcpy(&raw, encoded.data(), sizeof(int32_t)); + enc_val = static_cast(raw); + } else if (encoded.size() == sizeof(int64_t)) { + // INT64 physical type (precision <= 18) + int64_t raw; + std::memcpy(&raw, encoded.data(), sizeof(int64_t)); + enc_val = static_cast(raw); + } else { + // FIXED_LEN_BYTE_ARRAY / BYTE_ARRAY: big-endian two's complement. + // Defer to Decimal::FromUnscaledBytes so endianness, padding, and + // sign extension stay consistent with parquet_stats_extractor. + if (encoded.empty()) { + return std::nullopt; + } + Bytes bytes(encoded, GetDefaultPool().get()); + enc_val = + Decimal::FromUnscaledBytes(lit_decimal.Precision(), lit_decimal.Scale(), &bytes) + .Value(); + } + + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + default: + // TIMESTAMP, etc. - not yet supported for page-level filtering. + // TIMESTAMP is blocked at predicate_converter level (returns NotImplemented). + // Return nullopt to fall back to safe behavior (include page). + return std::nullopt; + } +} + +bool ColumnIndexFilter::PageMightContainEqual(const std::string& encoded_min, + const std::string& encoded_max, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; // Null is handled separately via null_pages + } + + // Page might contain equal if min <= literal <= max + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; // Can't compare, assume match + } + if (*cmp_min > 0) { + return false; // min > literal + } + + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + if (*cmp_max < 0) { + return false; // max < literal + } + + return true; // min <= literal <= max +} + +bool ColumnIndexFilter::PageMightContainLessThan(const std::string& encoded_min, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values < literal if min < literal + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; + } + return *cmp_min < 0; +} + +bool ColumnIndexFilter::PageMightContainLessOrEqual(const std::string& encoded_min, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values <= literal if min <= literal + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; + } + return *cmp_min <= 0; +} + +bool ColumnIndexFilter::PageMightContainGreaterThan(const std::string& encoded_max, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values > literal if max > literal + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + return *cmp_max > 0; +} + +bool ColumnIndexFilter::PageMightContainGreaterOrEqual(const std::string& encoded_max, + const Literal& literal, + FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values >= literal if max >= literal + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + return *cmp_max >= 0; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/column_index_filter.h b/src/paimon/format/parquet/column_index_filter.h new file mode 100644 index 00000000..56bb816e --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter.h @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/defs.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" +#include "parquet/page_index.h" + +namespace paimon { +class CompoundPredicate; +class LeafPredicate; +class Literal; +} // namespace paimon + +namespace paimon::parquet { + +/// ColumnIndexFilter calculates row ranges based on ColumnIndex statistics. +/// It uses the min/max values in the column index to determine which pages +/// might contain rows matching the predicate. +/// +/// The computed RowRanges serve two purposes: +/// 1. Row-group elimination: if no pages match, the entire row group is skipped. +/// 2. Page-level skipping: for partially matched row groups, RowRanges are passed +/// to PageFilteredRowGroupReader which uses data_page_filter to skip +/// non-matching pages at the I/O level, and SkipRecords/ReadRecords to skip +/// non-matching rows at the decode level within kept pages. +class ColumnIndexFilter { + public: + ColumnIndexFilter() = delete; + + /// Calculate row ranges based on predicate and column indices. + /// @param predicate The predicate to evaluate. + /// @param page_index_reader The page index reader for the file. + /// @param column_name_to_index Map from column name to column index. + /// @param row_group_index The row group index to filter. + /// @param row_group_row_count The number of rows in the row group. + /// @return RowRanges that may contain matching rows. + static Result CalculateRowRanges( + const std::shared_ptr& predicate, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + const std::map& column_name_to_index, int32_t row_group_index, + int64_t row_group_row_count); + + private: + /// Visit a predicate and calculate row ranges. + static Result VisitPredicate( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Visit a leaf predicate and calculate row ranges. + static Result VisitLeafPredicate( + const std::shared_ptr& leaf_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Visit a compound predicate (AND/OR) and calculate row ranges. + static Result VisitCompoundPredicate( + const std::shared_ptr& compound_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Filter pages based on column index statistics for EQUAL predicate. + static std::vector FilterPagesByEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for NOT_EQUAL predicate. + static std::vector FilterPagesByNotEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for LESS_THAN predicate. + static std::vector FilterPagesByLessThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for LESS_OR_EQUAL predicate. + static std::vector FilterPagesByLessOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for GREATER_THAN predicate. + static std::vector FilterPagesByGreaterThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for GREATER_OR_EQUAL predicate. + static std::vector FilterPagesByGreaterOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for IS_NULL predicate. + static std::vector FilterPagesByIsNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index); + + /// Filter pages based on column index statistics for IS_NOT_NULL predicate. + static std::vector FilterPagesByIsNotNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index); + + /// Filter pages based on column index statistics for IN predicate. + static std::vector FilterPagesByIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals, FieldType field_type); + + /// Filter pages based on column index statistics for NOT_IN predicate. + static std::vector FilterPagesByNotIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals); + + /// Build row ranges from page indices (must be sorted in ascending order). + static RowRanges BuildRowRangesFromPageIndices( + const std::vector& page_indices, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); + + /// Compare a parquet encoded value with a Literal. + /// @return -1 if encoded < literal, 0 if equal, 1 if encoded > literal. + /// nullopt if comparison cannot be performed (unsupported type, etc.). + static std::optional CompareEncodedWithLiteral(const std::string& encoded, + const Literal& literal, + FieldType field_type); + + /// Check if a page might contain a value equal to the literal. + /// Condition: min <= literal <= max + static bool PageMightContainEqual(const std::string& encoded_min, + const std::string& encoded_max, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values less than the literal. + /// Condition: min < literal + static bool PageMightContainLessThan(const std::string& encoded_min, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values less than or equal to the literal. + /// Condition: min <= literal + static bool PageMightContainLessOrEqual(const std::string& encoded_min, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values greater than the literal. + /// Condition: max > literal + static bool PageMightContainGreaterThan(const std::string& encoded_max, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values greater than or equal to the literal. + /// Condition: max >= literal + static bool PageMightContainGreaterOrEqual(const std::string& encoded_max, + const Literal& literal, FieldType field_type); +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp new file mode 100644 index 00000000..35e02576 --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -0,0 +1,658 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/column_index_filter.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/predicate/equal.h" +#include "paimon/common/predicate/in.h" +#include "paimon/common/predicate/leaf_predicate_impl.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/defs.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" +#include "parquet/file_reader.h" + +namespace paimon::parquet::test { + +// ===================================================================== +// RowRanges unit tests +// ===================================================================== + +class RowRangesTest : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(RowRangesTest, TestCreateSingle) { + RowRanges ranges = RowRanges::CreateSingle(100); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(100, ranges.RowCount()); + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +TEST_F(RowRangesTest, TestCreateEmpty) { + RowRanges ranges = RowRanges::CreateEmpty(); + EXPECT_TRUE(ranges.IsEmpty()); + EXPECT_EQ(0, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges().size()); +} + +TEST_F(RowRangesTest, TestAddRange) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(11, ranges.RowCount()); + EXPECT_EQ(1, ranges.GetRanges().size()); +} + +TEST_F(RowRangesTest, TestAddOverlappingRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(15, 25)); // overlaps with [10, 20] + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(25, ranges.GetRanges()[0].to); + EXPECT_EQ(16, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestAddAdjacentRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(21, 30)); // adjacent to [10, 20] + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(30, ranges.GetRanges()[0].to); + EXPECT_EQ(21, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestAddNonOverlappingRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + EXPECT_EQ(2, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(20, ranges.GetRanges()[0].to); + EXPECT_EQ(30, ranges.GetRanges()[1].from); + EXPECT_EQ(40, ranges.GetRanges()[1].to); + EXPECT_EQ(22, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestUnion) { + RowRanges left; + left.Add(RowRanges::Range(10, 20)); + left.Add(RowRanges::Range(40, 50)); + + RowRanges right; + right.Add(RowRanges::Range(15, 25)); + right.Add(RowRanges::Range(60, 70)); + + RowRanges result = RowRanges::Union(left, right); + EXPECT_EQ(3, result.GetRanges().size()); + EXPECT_EQ(10, result.GetRanges()[0].from); + EXPECT_EQ(25, result.GetRanges()[0].to); + EXPECT_EQ(40, result.GetRanges()[1].from); + EXPECT_EQ(50, result.GetRanges()[1].to); + EXPECT_EQ(60, result.GetRanges()[2].from); + EXPECT_EQ(70, result.GetRanges()[2].to); +} + +TEST_F(RowRangesTest, TestUnionWithOverlap) { + RowRanges left; + left.Add(RowRanges::Range(10, 30)); + + RowRanges right; + right.Add(RowRanges::Range(20, 40)); + + RowRanges result = RowRanges::Union(left, right); + EXPECT_EQ(1, result.GetRanges().size()); + EXPECT_EQ(10, result.GetRanges()[0].from); + EXPECT_EQ(40, result.GetRanges()[0].to); +} + +TEST_F(RowRangesTest, TestIntersection) { + RowRanges left; + left.Add(RowRanges::Range(10, 30)); + left.Add(RowRanges::Range(50, 70)); + + RowRanges right; + right.Add(RowRanges::Range(20, 40)); + right.Add(RowRanges::Range(60, 80)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_EQ(2, result.GetRanges().size()); + EXPECT_EQ(20, result.GetRanges()[0].from); + EXPECT_EQ(30, result.GetRanges()[0].to); + EXPECT_EQ(60, result.GetRanges()[1].from); + EXPECT_EQ(70, result.GetRanges()[1].to); +} + +TEST_F(RowRangesTest, TestIntersectionNoOverlap) { + RowRanges left; + left.Add(RowRanges::Range(10, 20)); + + RowRanges right; + right.Add(RowRanges::Range(30, 40)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_TRUE(result.IsEmpty()); +} + +TEST_F(RowRangesTest, TestIntersectionEmptyLeft) { + RowRanges left = RowRanges::CreateEmpty(); + + RowRanges right; + right.Add(RowRanges::Range(10, 20)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_TRUE(result.IsEmpty()); +} + +TEST_F(RowRangesTest, TestIsOverlapping) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + + EXPECT_TRUE(ranges.IsOverlapping(10, 20)); + EXPECT_TRUE(ranges.IsOverlapping(15, 25)); + EXPECT_TRUE(ranges.IsOverlapping(30, 40)); + EXPECT_FALSE(ranges.IsOverlapping(21, 29)); + EXPECT_FALSE(ranges.IsOverlapping(5, 9)); + EXPECT_FALSE(ranges.IsOverlapping(41, 50)); +} + +TEST_F(RowRangesTest, TestRowCount) { + RowRanges ranges; + ranges.Add(RowRanges::Range(0, 9)); + ranges.Add(RowRanges::Range(20, 29)); + EXPECT_EQ(20, ranges.RowCount()); + + ranges.Add(RowRanges::Range(10, 19)); // Fill the gap + EXPECT_EQ(30, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestToString) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + EXPECT_EQ("[[10, 20], [30, 40]]", ranges.ToString()); +} + +TEST_F(RowRangesTest, TestRangeOperations) { + RowRanges::Range r1(10, 20); + RowRanges::Range r2(30, 40); + RowRanges::Range r3(15, 25); + + // r1 lies entirely before r2; r3 overlaps r1. + EXPECT_TRUE(r1.to < r2.from); + EXPECT_FALSE(r1.from > r2.to); + EXPECT_FALSE(r1.to < r3.from); + EXPECT_FALSE(r1.from > r3.to); + EXPECT_EQ(11, r1.Count()); +} + +// ===================================================================== +// ColumnIndexFilter integration tests +// ===================================================================== + +/// Test fixture that creates real Parquet files with page index for testing +/// ColumnIndexFilter::CalculateRowRanges end-to-end. +/// +/// Data layout: 100 rows, 10 pages of 10 rows each. +/// Page 0: val [0, 9] +/// Page 1: val [10, 19] +/// ... +/// Page 9: val [90, 99] +class ColumnIndexFilterTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + + // Write the test file once for all tests + file_name_ = dir_->Str() + "/col_index_filter.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name_, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Open as raw ParquetFileReader + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + parquet_reader_ = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader_); + + page_index_reader_ = parquet_reader_->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader_); + + column_name_to_index_["val"] = 0; + row_group_row_count_ = parquet_reader_->metadata()->RowGroup(0)->num_rows(); + } + + static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(i); + } + auto array = builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + return arrow::StructArray::Make({array}, {field}).ValueOrDie(); + } + + void WriteTestFile(const std::string& file_name, + const std::shared_ptr& struct_array, + int32_t write_batch_size, int64_t max_row_group_length) { + auto data_type = struct_array->struct_type(); + auto data_schema = arrow::schema(data_type->fields()); + auto data_arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*struct_array, data_arrow_array.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_name, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder wp_builder; + wp_builder.write_batch_size(write_batch_size); + wp_builder.max_row_group_length(max_row_group_length); + wp_builder.disable_dictionary(); + wp_builder.enable_write_page_index(); + wp_builder.data_pagesize(1); + auto writer_properties = wp_builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, data_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(data_arrow_array.get())); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Close()); + } + + Result Filter(const std::shared_ptr& predicate) { + return ColumnIndexFilter::CalculateRowRanges(predicate, page_index_reader_, + column_name_to_index_, /*row_group_index=*/0, + row_group_row_count_); + } + + std::shared_ptr arrow_pool_; + std::shared_ptr pool_; + std::shared_ptr fs_; + std::unique_ptr dir_; + std::string file_name_; + std::unique_ptr<::parquet::ParquetFileReader> parquet_reader_; + std::shared_ptr<::parquet::PageIndexReader> page_index_reader_; + std::map column_name_to_index_; + int64_t row_group_row_count_ = 0; +}; + +/// EQUAL: val = 55 → should match only page 5 (rows [50,59]) +TEST_F(ColumnIndexFilterTest, EqualMatchSinglePage) { + auto pred = + PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(55))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Page 5 covers rows [50, 59] + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(50, ranges.GetRanges()[0].from); + EXPECT_EQ(59, ranges.GetRanges()[0].to); +} + +/// EQUAL: val = 0 → should match page 0 (rows [0,9]) +TEST_F(ColumnIndexFilterTest, EqualMatchFirstPage) { + auto pred = PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(0))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); +} + +/// EQUAL: val = 999 → should match no pages (value out of range) +TEST_F(ColumnIndexFilterTest, EqualNoMatch) { + auto pred = + PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(999))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// LESS_THAN: val < 25 → should match pages 0,1,2 (rows [0,29]) +/// Page 0: [0,9], Page 1: [10,19], Page 2: [20,29] — page 2 has min=20 < 25 +TEST_F(ColumnIndexFilterTest, LessThanMatchMultiplePages) { + auto pred = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(25))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Pages 0-2 match (min < 25) + EXPECT_EQ(30, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(29, ranges.GetRanges()[0].to); +} + +/// LESS_THAN: val < 0 → no pages match (min of page 0 is 0, which is not < 0) +TEST_F(ColumnIndexFilterTest, LessThanNoMatch) { + auto pred = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(0))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// GREATER_THAN: val > 85 → should match pages 8,9 +/// Page 8: max=89 > 85, Page 9: max=99 > 85 +TEST_F(ColumnIndexFilterTest, GreaterThanMatchLastPages) { + auto pred = + PredicateBuilder::GreaterThan(0, "val", FieldType::INT, Literal(static_cast(85))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(20, ranges.RowCount()); + EXPECT_EQ(80, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +/// GREATER_THAN: val > 99 → no pages match +TEST_F(ColumnIndexFilterTest, GreaterThanNoMatch) { + auto pred = + PredicateBuilder::GreaterThan(0, "val", FieldType::INT, Literal(static_cast(99))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// LESS_OR_EQUAL: val <= 9 → page 0 only (max=9 <= 9, but page 1 min=10 > 9) +TEST_F(ColumnIndexFilterTest, LessOrEqualBoundary) { + auto pred = + PredicateBuilder::LessOrEqual(0, "val", FieldType::INT, Literal(static_cast(9))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); +} + +/// GREATER_OR_EQUAL: val >= 90 → page 9 only +TEST_F(ColumnIndexFilterTest, GreaterOrEqualBoundary) { + auto pred = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(90))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(90, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +/// IN: val IN (5, 55, 95) → pages 0, 5, 9 +TEST_F(ColumnIndexFilterTest, InMatchMultiplePages) { + auto pred = + PredicateBuilder::In(0, "val", FieldType::INT, + {Literal(static_cast(5)), Literal(static_cast(55)), + Literal(static_cast(95))}); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Pages 0, 5, 9 + EXPECT_EQ(3, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); + EXPECT_EQ(50, ranges.GetRanges()[1].from); + EXPECT_EQ(59, ranges.GetRanges()[1].to); + EXPECT_EQ(90, ranges.GetRanges()[2].from); + EXPECT_EQ(99, ranges.GetRanges()[2].to); +} + +/// IN: val IN (999) → no match +TEST_F(ColumnIndexFilterTest, InNoMatch) { + auto pred = + PredicateBuilder::In(0, "val", FieldType::INT, {Literal(static_cast(999))}); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// IS_NOT_NULL on non-nullable column → all pages match +TEST_F(ColumnIndexFilterTest, IsNotNullAllPages) { + auto pred = PredicateBuilder::IsNotNull(0, "val", FieldType::INT); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(row_group_row_count_, ranges.RowCount()); +} + +/// AND: val >= 30 AND val < 50 → pages 3, 4 +TEST_F(ColumnIndexFilterTest, AndCompound) { + auto ge = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(30))); + auto lt = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(50))); + ASSERT_OK_AND_ASSIGN(auto pred, PredicateBuilder::And({ge, lt})); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(20, ranges.RowCount()); + EXPECT_EQ(30, ranges.GetRanges()[0].from); + EXPECT_EQ(49, ranges.GetRanges()[0].to); +} + +/// OR: val < 10 OR val >= 90 → pages 0, 9 +TEST_F(ColumnIndexFilterTest, OrCompound) { + auto lt = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(10))); + auto ge = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(90))); + ASSERT_OK_AND_ASSIGN(auto pred, PredicateBuilder::Or({lt, ge})); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(2, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); + EXPECT_EQ(90, ranges.GetRanges()[1].from); + EXPECT_EQ(99, ranges.GetRanges()[1].to); +} + +TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) { + for (FieldType field_type : {FieldType::FLOAT, FieldType::DOUBLE}) { + std::shared_ptr values; + if (field_type == FieldType::FLOAT) { + arrow::FloatBuilder builder; + ASSERT_TRUE(builder.Reserve(30).ok()); + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(-0.0f); + } + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(0.0f); + } + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(1.0f); + } + values = builder.Finish().ValueOrDie(); + } else { + arrow::DoubleBuilder builder; + ASSERT_TRUE(builder.Reserve(30).ok()); + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(-0.0); + } + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(0.0); + } + for (int32_t i = 0; i < 10; ++i) { + builder.UnsafeAppend(1.0); + } + values = builder.Finish().ValueOrDie(); + } + + auto field = arrow::field( + "value", field_type == FieldType::FLOAT ? arrow::float32() : arrow::float64()); + auto data = arrow::StructArray::Make({values}, {field}).ValueOrDie(); + std::string file_name = + dir_->Str() + (field_type == FieldType::FLOAT ? "/float_signed_zero.parquet" + : "/double_signed_zero.parquet"); + WriteTestFile(file_name, data, /*write_batch_size=*/10, + /*max_row_group_length=*/30); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(reader); + auto page_index_reader = reader->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader); + auto row_group_page_index = page_index_reader->RowGroup(0); + ASSERT_TRUE(row_group_page_index); + auto column_index = row_group_page_index->GetColumnIndex(0); + ASSERT_TRUE(column_index); + // Pages 0 and 1 contain only -0.0 and +0.0 respectively. Parquet normalizes both + // zero-only page bounds to [-0.0, +0.0], so page pruning must remain a safe superset. + ASSERT_EQ(3, column_index->encoded_min_values().size()); + ASSERT_EQ(3, column_index->encoded_max_values().size()); + for (int32_t page_index : {0, 1}) { + if (field_type == FieldType::FLOAT) { + float min_value; + float max_value; + std::memcpy(&min_value, column_index->encoded_min_values()[page_index].data(), + sizeof(float)); + std::memcpy(&max_value, column_index->encoded_max_values()[page_index].data(), + sizeof(float)); + ASSERT_TRUE(std::signbit(min_value)); + ASSERT_FALSE(std::signbit(max_value)); + } else { + double min_value; + double max_value; + std::memcpy(&min_value, column_index->encoded_min_values()[page_index].data(), + sizeof(double)); + std::memcpy(&max_value, column_index->encoded_max_values()[page_index].data(), + sizeof(double)); + ASSERT_TRUE(std::signbit(min_value)); + ASSERT_FALSE(std::signbit(max_value)); + } + } + + auto less_negative_zero = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); + ASSERT_OK_AND_ASSIGN( + auto ranges, ColumnIndexFilter::CalculateRowRanges( + less_negative_zero, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_TRUE(ranges.IsEmpty()) << "field type: " << static_cast(field_type); + + auto less_positive_zero = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(0.0f) : Literal(0.0)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + less_positive_zero, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_EQ(20, ranges.RowCount()); + ASSERT_EQ(1, ranges.GetRanges().size()); + ASSERT_EQ(0, ranges.GetRanges()[0].from); + ASSERT_EQ(19, ranges.GetRanges()[0].to); + + auto greater_negative_zero = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + greater_negative_zero, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_EQ(30, ranges.RowCount()); + + auto not_equal_negative_zero = PredicateBuilder::NotEqual( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + not_equal_negative_zero, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_EQ(30, ranges.RowCount()); + + auto greater_finite = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + greater_finite, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_TRUE(ranges.IsEmpty()); + + auto greater_between_pages = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(0.5f) : Literal(0.5)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + greater_between_pages, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_EQ(10, ranges.RowCount()); + ASSERT_EQ(1, ranges.GetRanges().size()); + ASSERT_EQ(20, ranges.GetRanges()[0].from); + ASSERT_EQ(29, ranges.GetRanges()[0].to); + + auto equal_finite = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"value", field_type, + field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0)); + ASSERT_OK_AND_ASSIGN( + ranges, ColumnIndexFilter::CalculateRowRanges( + equal_finite, page_index_reader, {{"value", 0}}, + /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows())); + ASSERT_TRUE(ranges.IsEmpty()); + } +} + +/// Predicates referencing fields absent from the data file are stripped upstream +/// by FieldMappingBuilder, so reaching ColumnIndexFilter with such a predicate is +/// a contract violation and surfaces as an error. +TEST_F(ColumnIndexFilterTest, UnknownColumnReturnsError) { + auto pred = PredicateBuilder::Equal(0, "nonexistent", FieldType::INT, + Literal(static_cast(42))); + EXPECT_FALSE(Filter(pred).ok()); +} + +/// Null predicate → all rows +TEST_F(ColumnIndexFilterTest, NullPredicateReturnsAllRows) { + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(nullptr)); + EXPECT_EQ(row_group_row_count_, ranges.RowCount()); +} + +/// Predicates other than IsNull/IsNotNull are not allowed without a literal. +/// PredicateBuilder (public API) does not support constructing them without +/// a literal, so the filter should return an error for this invalid input. +TEST_F(ColumnIndexFilterTest, EmptyLiteralsReturnsError) { + auto pred = std::make_shared(paimon::Equal::Instance(), 0, "val", + FieldType::INT, std::vector()); + auto result = Filter(pred); + EXPECT_FALSE(result.ok()); +} + +/// Empty literals for IN predicate — same rule applies: non-IS_NULL/IS_NOT_NULL +/// predicates without literals are invalid and should return an error. +TEST_F(ColumnIndexFilterTest, EmptyLiteralsInReturnsError) { + auto pred = std::make_shared(paimon::In::Instance(), 0, "val", + FieldType::INT, std::vector()); + auto result = Filter(pred); + EXPECT_FALSE(result.ok()); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 674e3c62..39664658 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -18,114 +18,311 @@ #include "paimon/format/parquet/file_reader_wrapper.h" +#include #include #include +#include "arrow/io/interfaces.h" #include "arrow/record_batch.h" #include "arrow/util/range.h" #include "fmt/format.h" +#include "paimon/common/utils/arrow/arrow_utils.h" +#include "paimon/format/parquet/column_index_filter.h" +#include "paimon/format/parquet/page_filtered_row_group_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/macros.h" #include "parquet/arrow/reader.h" +#include "parquet/arrow/schema.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" +#include "parquet/page_index.h" namespace paimon::parquet { +namespace { + +// Merge overlapping or adjacent ReadRanges into a minimal set of non-overlapping ranges. +// PreBufferRanges requires non-overlapping ranges, so this is necessary when combining +// ranges from multiple sources (page-level ranges, column chunk ranges, etc.). +std::vector<::arrow::io::ReadRange> MergeOverlappingRanges( + std::vector<::arrow::io::ReadRange> ranges) { + if (ranges.empty()) { + return ranges; + } + + // Sort by offset + std::sort(ranges.begin(), ranges.end(), + [](const ::arrow::io::ReadRange& a, const ::arrow::io::ReadRange& b) { + return a.offset < b.offset; + }); + + std::vector<::arrow::io::ReadRange> merged; + merged.push_back(ranges[0]); + + for (size_t i = 1; i < ranges.size(); ++i) { + auto& last = merged.back(); + const auto& curr = ranges[i]; + // Check if current range overlaps or is adjacent to the last merged range + int64_t last_end = last.offset + last.length; + if (curr.offset <= last_end) { + // Merge: extend the last range if current extends beyond it + int64_t curr_end = curr.offset + curr.length; + if (curr_end > last_end) { + last.length = curr_end - last.offset; + } + } else { + // No overlap, add as new range + merged.push_back(curr); + } + } + + return merged; +} + +} // namespace + Result> FileReaderWrapper::Create( - std::unique_ptr<::parquet::arrow::FileReader>&& file_reader) { - if (file_reader == nullptr) { - return Status::Invalid("file reader wrapper create failed. file reader is nullptr"); - } - std::vector> all_row_group_ranges; - auto meta_data = file_reader->parquet_reader()->metadata(); - // prepare [start_row_idx, end_row_idx) for all row groups - uint64_t start_row_idx = 0; - for (int32_t i = 0; i < meta_data->num_row_groups(); i++) { - uint64_t end_row_idx = start_row_idx + meta_data->RowGroup(i)->num_rows(); - all_row_group_ranges.emplace_back(start_row_idx, end_row_idx); - start_row_idx = end_row_idx; - } - uint64_t num_rows = file_reader->parquet_reader()->metadata()->num_rows(); - if (start_row_idx != num_rows) { - assert(false); - return Status::Invalid( - fmt::format("unexpected error. row group ranges not match with num rows {}", num_rows)); + std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, int64_t batch_size, + std::shared_ptr<::arrow::MemoryPool> pool) { + try { + if (file_reader == nullptr) { + return Status::Invalid("file reader wrapper create failed. file reader is nullptr"); + } + std::vector> all_row_group_ranges; + auto meta_data = file_reader->parquet_reader()->metadata(); + // prepare [start_row_idx, end_row_idx) for all row groups + uint64_t start_row_idx = 0; + for (int32_t i = 0; i < meta_data->num_row_groups(); i++) { + uint64_t end_row_idx = start_row_idx + meta_data->RowGroup(i)->num_rows(); + all_row_group_ranges.emplace_back(start_row_idx, end_row_idx); + start_row_idx = end_row_idx; + } + uint64_t num_rows = file_reader->parquet_reader()->metadata()->num_rows(); + if (start_row_idx != num_rows) { + assert(false); + return Status::Invalid(fmt::format( + "unexpected error. row group ranges not match with num rows {}", num_rows)); + } + std::vector columns_indices = + arrow::internal::Iota(file_reader->parquet_reader()->metadata()->num_columns()); + auto file_reader_wrapper = std::unique_ptr(new FileReaderWrapper( + std::move(file_reader), all_row_group_ranges, num_rows, batch_size, pool)); + std::vector all_target_row_groups; + for (int32_t i = 0; i < file_reader_wrapper->GetNumberOfRowGroups(); i++) { + all_target_row_groups.emplace_back(/*rg_index=*/i, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()); + } + PAIMON_RETURN_NOT_OK( + file_reader_wrapper->PrepareForReadingLazy(all_target_row_groups, columns_indices)); + return file_reader_wrapper; } - std::vector row_groups_indices = arrow::internal::Iota(file_reader->num_row_groups()); - std::vector columns_indices = - arrow::internal::Iota(file_reader->parquet_reader()->metadata()->num_columns()); - auto file_reader_wrapper = std::unique_ptr( - new FileReaderWrapper(std::move(file_reader), all_row_group_ranges, num_rows)); - PAIMON_RETURN_NOT_OK(file_reader_wrapper->PrepareForReadingLazy( - std::set(row_groups_indices.begin(), row_groups_indices.end()), columns_indices)); - return file_reader_wrapper; + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Create") +} + +FileReaderWrapper::~FileReaderWrapper() { + WaitForPendingPreBuffer(); +} + +Result> FileReaderWrapper::GetSchema() const { + try { + std::shared_ptr file_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&file_schema)); + return file_schema; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::GetSchema") +} + +Status FileReaderWrapper::Close() { + try { + if (batch_reader_) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(batch_reader_->Close()); + } + return Status::OK(); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Close") } FileReaderWrapper::FileReaderWrapper( std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, - const std::vector>& all_row_group_ranges, uint64_t num_rows) + const std::vector>& all_row_group_ranges, uint64_t num_rows, + int64_t batch_size, std::shared_ptr<::arrow::MemoryPool> pool) : file_reader_(std::move(file_reader)), all_row_group_ranges_(all_row_group_ranges), + pool_(std::move(pool)), + batch_size_(batch_size), num_rows_(num_rows) {} +void FileReaderWrapper::WaitForPendingPreBuffer() { + if (!prebuffered_ranges_.empty() && file_reader_) { + // Wait for all outstanding PreBuffer async reads to complete before destruction. + // Without this, JindoSDK async pread callbacks may fire after the underlying + // buffers and memory pool are freed, causing use-after-free crashes. + auto status = + file_reader_->parquet_reader()->WhenBufferedRanges(prebuffered_ranges_).status(); + (void)status; // Best-effort; ignore errors during cleanup + prebuffered_ranges_.clear(); + } +} + +void FileReaderWrapper::AdvanceToNextRowGroup() { + current_row_group_idx_++; + // Skip row groups excluded by read range. + while (current_row_group_idx_ < target_row_groups_.size() && + target_row_groups_[current_row_group_idx_].IsExcludedByReadRange()) { + current_row_group_idx_++; + } + if (current_row_group_idx_ >= target_row_groups_.size()) { + next_row_to_read_ = num_rows_; + } else { + next_row_to_read_ = + all_row_group_ranges_[target_row_groups_[current_row_group_idx_].GetRowGroupIndex()] + .first; + } +} + Status FileReaderWrapper::SeekToRow(uint64_t row_number) { - for (uint64_t i = 0; i < target_row_groups_.size(); i++) { - if (row_number > target_row_groups_[i].first && row_number < target_row_groups_[i].second) { - return Status::Invalid(fmt::format( - "seek to row failed. row number {} should not be in the middle of readable range", - row_number)); - } - if (target_row_groups_[i].first >= row_number) { - current_row_group_idx_ = i; - next_row_to_read_ = target_row_groups_[i].first; - std::vector target_row_group_indices; - for (uint64_t j = i; j < target_row_groups_.size(); j++) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); + try { + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; + + for (uint64_t i = 0; i < target_row_groups_.size(); i++) { + if (target_row_groups_[i].IsExcludedByReadRange()) { + continue; + } + int32_t rg_id = target_row_groups_[i].GetRowGroupIndex(); + uint64_t rg_start = all_row_group_ranges_[rg_id].first; + uint64_t rg_end = all_row_group_ranges_[rg_id].second; + if (row_number > rg_start && row_number < rg_end) { + return Status::Invalid( + fmt::format("seek to row failed. row number {} should not be in the middle of " + "readable range", + row_number)); + } + if (rg_start >= row_number) { + current_row_group_idx_ = i; + next_row_to_read_ = rg_start; + + // Rebuild batch_reader_ for non-page-filtered RGs at/after seek position. + std::vector fully_matched_indices; + for (uint64_t j = i; j < target_row_groups_.size(); j++) { + if (!target_row_groups_[j].IsExcludedByReadRange() && + !target_row_groups_[j].IsPartiallyMatched()) { + fully_matched_indices.push_back(target_row_groups_[j].GetRowGroupIndex()); + } + } + if (!fully_matched_indices.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( + fully_matched_indices, target_column_indices_, &batch_reader_)); + } else { + batch_reader_.reset(); + } + return Status::OK(); } - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - target_row_group_indices, target_column_indices_, &batch_reader_)); - return Status::OK(); } + next_row_to_read_ = num_rows_; + current_row_group_idx_ = target_row_groups_.size(); + return Status::OK(); } - next_row_to_read_ = num_rows_; - current_row_group_idx_ = target_row_groups_.size(); - return Status::OK(); + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::SeekToRow") } -Result> FileReaderWrapper::Next() { - if (PAIMON_UNLIKELY(!reader_initialized_)) { - PAIMON_RETURN_NOT_OK(PrepareForReading(target_row_group_indices_, target_column_indices_)); +Result> FileReaderWrapper::NextPageFiltered() { + int32_t rg_id = target_row_groups_[current_row_group_idx_].GetRowGroupIndex(); + + // Construct the per-RG streaming reader on demand. + if (!current_page_filtered_reader_) { + const auto& target_rg = target_row_groups_[current_row_group_idx_]; + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + target_rg, target_column_indices_, file_reader_->parquet_reader()); + bool pre_buffered = !prebuffered_ranges_.empty(); + int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); + PAIMON_ASSIGN_OR_RAISE( + current_page_filtered_reader_, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + target_rg, target_column_indices_, file_reader_->properties().cache_options(), + pre_buffered, page_ranges, max_chunksize, pool_, file_reader_.get())); + current_filtered_row_ranges_ = target_rg.GetRowRanges(); + current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; + filtered_global_offset_ = 0; } + std::shared_ptr record_batch; - if (current_row_group_idx_ < target_row_groups_.size()) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch, batch_reader_->Next()); - } + PAIMON_RETURN_NOT_OK_FROM_ARROW(current_page_filtered_reader_->ReadNext(&record_batch)); + if (record_batch) { - int64_t num_rows = record_batch->num_rows(); - previous_first_row_ = next_row_to_read_; - if (next_row_to_read_ + num_rows < target_row_groups_[current_row_group_idx_].second) { - next_row_to_read_ += num_rows; - } else if (next_row_to_read_ + num_rows == - target_row_groups_[current_row_group_idx_].second) { - if (current_row_group_idx_ == target_row_groups_.size() - 1) { - // current row group is the last. - next_row_to_read_ = num_rows_; - } else { - current_row_group_idx_++; - next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; + auto original_row = + current_filtered_row_ranges_.MapFilteredIndexToOriginalRow(filtered_global_offset_); + previous_first_row_ = original_row.has_value() ? current_filtered_rg_start_ + + static_cast(*original_row) + : current_filtered_rg_start_; + filtered_global_offset_ += record_batch->num_rows(); + return record_batch; + } + + // RG exhausted — reset and advance. + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; + AdvanceToNextRowGroup(); + return std::shared_ptr(); +} + +Result> FileReaderWrapper::NextFullyMatched() { + if (!batch_reader_) { + return std::shared_ptr(); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr record_batch, + batch_reader_->Next()); + if (!record_batch) { + return std::shared_ptr(); + } + // Large binary columns (exceed 2GB) may split at different row boundaries. TableBatchReader + // aligns their chunks by slicing columns, which may leave non-zero child offsets. + PAIMON_ASSIGN_OR_RAISE(record_batch, + ArrowUtils::NormalizeRecordBatchOffsets(record_batch, pool_.get())); + + int32_t rg_id = target_row_groups_[current_row_group_idx_].GetRowGroupIndex(); + uint64_t rg_end = all_row_group_ranges_[rg_id].second; + int64_t num_rows = record_batch->num_rows(); + + previous_first_row_ = next_row_to_read_; + if (next_row_to_read_ + num_rows < rg_end) { + next_row_to_read_ += num_rows; + } else if (next_row_to_read_ + num_rows == rg_end) { + AdvanceToNextRowGroup(); + } else { + return Status::Invalid( + fmt::format("Next failed. next_row_to_read {} + num_rows {} exceeds row group end {}", + next_row_to_read_, num_rows, rg_end)); + } + return record_batch; +} + +Result> FileReaderWrapper::Next() { + try { + if (PAIMON_UNLIKELY(!reader_initialized_)) { + PAIMON_RETURN_NOT_OK(PrepareForReading(target_row_groups_, target_column_indices_)); + } + + while (current_row_group_idx_ < target_row_groups_.size()) { + bool is_partially_matched = + target_row_groups_[current_row_group_idx_].IsPartiallyMatched(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, + is_partially_matched ? NextPageFiltered() : NextFullyMatched()); + if (batch) { + return batch; + } else if (!is_partially_matched) { + // Null from fully-matched path means batch_reader_ is globally exhausted. + break; } - } else { - return Status::Invalid(fmt::format( - "Next failed. Unexpected error, next row to read {} + num rows just read {} " - "should always be within current row group range or exactly equals to current " - "row group end {}", - next_row_to_read_, num_rows, target_row_groups_[current_row_group_idx_].second)); + // current_row_group_idx_ has been advanced in NextPageFiltered() or NextFullyMatched(), + // loop to try next RG. } - } else { + previous_first_row_ = next_row_to_read_; + return std::shared_ptr(); } - return record_batch; + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Next") } Result>> FileReaderWrapper::GetRowGroupRanges( @@ -141,69 +338,177 @@ Result>> FileReaderWrapper::GetRowGrou return row_group_ranges; } -Status FileReaderWrapper::PrepareForReadingLazy(const std::set& target_row_group_indices, - const std::vector& column_indices) { - target_row_group_indices_ = target_row_group_indices; +Status FileReaderWrapper::PrepareForReadingLazy( + const std::vector& target_row_groups, + const std::vector& column_indices) { + target_row_groups_ = target_row_groups; target_column_indices_ = column_indices; reader_initialized_ = false; return Status::OK(); } -Status FileReaderWrapper::PrepareForReading(const std::set& target_row_group_indices, - const std::vector& column_indices) { - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - std::unique_ptr batch_reader; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), - column_indices, &batch_reader)); - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); - if (target_row_groups_.empty()) { - next_row_to_read_ = num_rows_; - } else { - next_row_to_read_ = target_row_groups_[0].first; +std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( + const std::vector& column_indices) { + std::vector<::arrow::io::ReadRange> ranges; + auto file_metadata = file_reader_->parquet_reader()->metadata(); + + for (const auto& trg : target_row_groups_) { + if (trg.IsExcludedByReadRange()) continue; + + if (trg.IsPartiallyMatched()) { + // Page-filtered RGs: only matching page byte ranges. + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + trg, column_indices, file_reader_->parquet_reader()); + ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()), + std::make_move_iterator(page_ranges.end())); + } else { + // Fully-matched RGs: entire column chunk ranges. + auto rg_metadata = file_metadata->RowGroup(trg.GetRowGroupIndex()); + for (int32_t col_idx : column_indices) { + auto col_chunk = rg_metadata->ColumnChunk(col_idx); + int64_t offset = col_chunk->data_page_offset(); + if (col_chunk->has_dictionary_page() && col_chunk->dictionary_page_offset() > 0 && + offset > col_chunk->dictionary_page_offset()) { + offset = col_chunk->dictionary_page_offset(); + } + ranges.push_back({offset, col_chunk->total_compressed_size()}); + } + } } - previous_first_row_ = std::numeric_limits::max(); - current_row_group_idx_ = 0; - reader_initialized_ = true; - return Status::OK(); + return ranges; } -Result> FileReaderWrapper::FilterRowGroupsByReadRanges( - const std::vector>& read_ranges, - const std::vector& src_row_groups) const { - std::set target_row_groups; - PAIMON_ASSIGN_OR_RAISE(std::set row_groups_to_read, - ReadRangesToRowGroupIds(read_ranges)); - for (const auto& row_group_id : src_row_groups) { - if (row_groups_to_read.find(row_group_id) != row_groups_to_read.end()) { - target_row_groups.emplace(row_group_id); +void FileReaderWrapper::DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges) { + const auto& cache_opts = file_reader_->properties().cache_options(); + ::arrow::io::IOContext io_ctx(pool_.get()); + auto merged_ranges = MergeOverlappingRanges(std::move(ranges)); + try { + file_reader_->parquet_reader()->PreBufferRanges(merged_ranges, io_ctx, cache_opts); + prebuffered_ranges_ = std::move(merged_ranges); + } catch (const std::exception&) { + prebuffered_ranges_.clear(); + } +} + +Status FileReaderWrapper::PrepareForReading(const std::vector& target_row_groups, + const std::vector& column_indices) { + try { + target_row_groups_ = target_row_groups; + target_column_indices_ = column_indices; + + // Partition into fully-matched and page-filtered row groups, skipping excluded ones. + std::vector fully_matched_row_groups; + uint64_t active_count = 0; + for (const auto& trg : target_row_groups_) { + if (trg.IsExcludedByReadRange()) { + continue; + } + active_count++; + if (!trg.IsPartiallyMatched()) { + fully_matched_row_groups.push_back(trg.GetRowGroupIndex()); + } + } + + bool has_partially_matched = fully_matched_row_groups.size() != active_count; + + WaitForPendingPreBuffer(); + + // TODO(Yonghao Fang): Neither Paimon nor Arrow manage the size and lifecycle of prebuffered + // caches. So when a lot of row is needed, there is possibility of OOM due to too much + // prebuffering. Also, DispatchPreBuffer will drop previous prebuffered ranges by + // GetRecordBatchReader, which cause IO wastes. + + // Create standard reader for fully-matched row groups. + if (!fully_matched_row_groups.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( + fully_matched_row_groups, column_indices, &batch_reader_)); + } else { + batch_reader_.reset(); + } + + // When page-filtered RGs exist, issue a single PreBuffer covering both kinds. + // Otherwise GetRecordBatchReader already issued PreBuffer internally. + if (has_partially_matched) { + auto all_ranges = CollectPreBufferRanges(column_indices); + DispatchPreBuffer(std::move(all_ranges)); + } + + // Reset read state. Find the first non-excluded row group. + uint64_t first_active_idx = 0; + while (first_active_idx < target_row_groups_.size() && + target_row_groups_[first_active_idx].IsExcludedByReadRange()) { + first_active_idx++; + } + if (first_active_idx >= target_row_groups_.size()) { + next_row_to_read_ = num_rows_; + } else { + next_row_to_read_ = + all_row_group_ranges_[target_row_groups_[first_active_idx].GetRowGroupIndex()] + .first; } + previous_first_row_ = std::numeric_limits::max(); + current_row_group_idx_ = first_active_idx; + reader_initialized_ = true; + return Status::OK(); } - return target_row_groups; + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::PrepareForReading") } -Result> FileReaderWrapper::ReadRangesToRowGroupIds( - const std::vector>& read_ranges) const { - std::set selected_row_group_ids; +Status FileReaderWrapper::ApplyReadRanges( + const std::vector>& read_ranges) { + if (read_ranges.empty()) { + for (auto& trg : target_row_groups_) { + trg.SetExcludedByReadRange(true); + } + reader_initialized_ = false; + return Status::OK(); + } + // Build a set of row group indices whose range matches one of the read ranges. + std::set matching_rg_indices; for (const auto& read_range : read_ranges) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(read_range)); - selected_row_group_ids.emplace(row_group_id); + for (size_t i = 0; i < all_row_group_ranges_.size(); i++) { + if (all_row_group_ranges_[i] == read_range) { + matching_rg_indices.insert(static_cast(i)); + } + } + } + // Mark each target row group as excluded or not based on the matching set. + for (auto& trg : target_row_groups_) { + trg.SetExcludedByReadRange(matching_rg_indices.count(trg.GetRowGroupIndex()) == 0); } - return selected_row_group_ids; + reader_initialized_ = false; + return Status::OK(); } -Result FileReaderWrapper::GetRowGroupId(std::pair target_range) const { - for (size_t i = 0; i < all_row_group_ranges_.size(); i++) { - if (all_row_group_ranges_[i] == target_range) { - return i; +std::shared_ptr<::parquet::PageIndexReader> FileReaderWrapper::GetPageIndexReader() { + try { + return file_reader_->parquet_reader()->GetPageIndexReader(); + } catch (...) { + // Page index is optional; degrade gracefully if the metadata read throws. + return nullptr; + } +} + +Result FileReaderWrapper::CalculateFilteredRowRanges( + int32_t row_group_index, const std::shared_ptr& predicate, + const std::map& column_name_to_index) { + try { + auto meta_data = file_reader_->parquet_reader()->metadata(); + int64_t row_count = meta_data->RowGroup(row_group_index)->num_rows(); + + if (!predicate) { + return RowRanges::CreateSingle(row_count); } + + auto page_index_reader = GetPageIndexReader(); + if (!page_index_reader) { + return RowRanges::CreateSingle(row_count); + } + + return ColumnIndexFilter::CalculateRowRanges( + predicate, page_index_reader, column_name_to_index, row_group_index, row_count); } - return Status::Invalid(fmt::format( - "not expected failure. target range bound '{},{}' not match with row group range bound", - target_range.first, target_range.second)); + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::CalculateFilteredRowRanges") } } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index f20e94e9..c62c1b96 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -28,102 +29,172 @@ #include "arrow/array.h" #include "arrow/compute/api.h" #include "arrow/dataset/file_parquet.h" +#include "arrow/io/caching.h" #include "arrow/record_batch.h" #include "arrow/type.h" #include "arrow/type_fwd.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" #include "paimon/result.h" #include "paimon/status.h" #include "parquet/arrow/reader.h" +#include "parquet/page_index.h" namespace arrow { class Schema; } // namespace arrow +namespace paimon { +class Predicate; +} // namespace paimon + namespace paimon::parquet { // The FileReaderWrapper is a decorator class designed to support seek functionality, as well as the // methods GetPreviousBatchFirstRowNumber and GetNextRowToRead. class FileReaderWrapper { public: + ~FileReaderWrapper(); + static Result> Create( - std::unique_ptr<::parquet::arrow::FileReader>&& reader); + std::unique_ptr<::parquet::arrow::FileReader>&& reader, int64_t batch_size, + std::shared_ptr pool); + /// Seek to the specified row number. + /// @param row_number The row to seek to (must be at a row group boundary). Status SeekToRow(uint64_t row_number); + /// Read the next batch of rows. + /// @return The next RecordBatch, or nullptr if end of data. Result> Next(); + /// Get the first row number of the previously returned batch. Result GetPreviousBatchFirstRowNumber() const { return previous_first_row_; } + /// Get the row number that will be read next. uint64_t GetNextRowToRead() const { return next_row_to_read_; } + /// Get the total number of rows in the file. uint64_t GetNumberOfRows() const { return num_rows_; } + /// Get the number of row groups in the file. int32_t GetNumberOfRowGroups() const { return file_reader_->num_row_groups(); } - ::parquet::arrow::FileReader* GetFileReader() const { + /// Get the underlying Parquet file reader. + ::parquet::arrow::FileReader* GetFileReader() { return file_reader_.get(); } + /// Get the [start, end) ranges for all row groups. const std::vector>& GetAllRowGroupRanges() const { return all_row_group_ranges_; } - Result> GetSchema() const { - std::shared_ptr file_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&file_schema)); - return file_schema; - } + /// Get the Arrow schema of the file. + Result> GetSchema() const; - Status Close() { - if (batch_reader_) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(batch_reader_->Close()); - } - return Status::OK(); - } + /// Close the batch reader and release resources. + Status Close(); + /// Get the [start, end) ranges for the specified row groups. + /// @param row_group_indices The row group indices to get ranges for. Result>> GetRowGroupRanges( const std::set& row_group_indices) const; - Status PrepareForReadingLazy(const std::set& row_group_indices, + /// Prepare for lazy reading of the specified row groups and columns. + /// Actual reader initialization is deferred until the first Next() call. + Status PrepareForReadingLazy(const std::vector& target_row_groups, const std::vector& column_indices); - Status PrepareForReading(const std::set& row_group_indices, + + /// Prepare for immediate reading of the specified row groups and columns. + /// Initializes the reader and starts pre-buffering I/O. + /// + /// Note: when the read schema has nested sub-field projection, + /// page-level filtering is disabled temporarily due to known offset + /// calculation issues for nested pages. + Status PrepareForReading(const std::vector& target_row_groups, const std::vector& column_indices); - Result> FilterRowGroupsByReadRanges( - const std::vector>& read_ranges, - const std::vector& src_row_groups) const; + /// Apply read ranges to the current target_row_groups_, keeping only those + /// whose row-group range is equal to one of the given read ranges. + /// Resets reader state so that the next Next() call will re-initialize. + Status ApplyReadRanges(const std::vector>& read_ranges); + + /// Get the page index reader for the file. + /// Returns nullptr if page index is not available. + std::shared_ptr<::parquet::PageIndexReader> GetPageIndexReader(); + + /// Calculate filtered row ranges for a row group based on predicate. + /// @param row_group_index The row group index. + /// @param predicate The predicate to evaluate. + /// @param column_name_to_index Map from column name to column index. + /// @return RowRanges that may contain matching rows. + Result CalculateFilteredRowRanges( + int32_t row_group_index, const std::shared_ptr& predicate, + const std::map& column_name_to_index); private: FileReaderWrapper(std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, const std::vector>& all_row_group_ranges, - uint64_t num_rows); + uint64_t num_rows, int64_t batch_size, + std::shared_ptr<::arrow::MemoryPool> pool); + + /// Wait for all pending PreBuffer operations to complete. + void WaitForPendingPreBuffer(); - Result> ReadRangesToRowGroupIds( - const std::vector>& read_ranges) const; - Result GetRowGroupId(std::pair target_range) const; + /// Advance current_row_group_idx_ to the next row group and update next_row_to_read_. + void AdvanceToNextRowGroup(); + + /// Read next batch from a page-filtered row group. Returns nullptr when the RG is exhausted. + Result> NextPageFiltered(); + + /// Read next batch from the fully-matched batch_reader_. Returns nullptr when exhausted. + Result> NextFullyMatched(); + + /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched). + std::vector<::arrow::io::ReadRange> CollectPreBufferRanges( + const std::vector& column_indices); + + /// Dispatch a single PreBufferRanges call with merged ranges. + void DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges); std::unique_ptr<::parquet::arrow::FileReader> file_reader_; std::unique_ptr batch_reader_; std::vector> all_row_group_ranges_; - std::set target_row_group_indices_; - std::vector> target_row_groups_; std::vector target_column_indices_; + std::shared_ptr<::arrow::MemoryPool> pool_; + int64_t batch_size_; // 0 means no limit + const uint64_t num_rows_; uint64_t next_row_to_read_ = std::numeric_limits::max(); uint64_t previous_first_row_ = std::numeric_limits::max(); uint64_t current_row_group_idx_ = 0; bool reader_initialized_ = false; + + // Streaming reader for the currently-active page-filtered row group. Created lazily + // on the first Next() call into a page-filtered RG, drained batch-by-batch, then reset + // when ReadNext returns nullptr (end of that RG). + std::unique_ptr current_page_filtered_reader_; + int64_t filtered_global_offset_ = 0; // Cumulative filtered-row offset within RG + RowRanges current_filtered_row_ranges_; // RowRanges for the active page-filtered RG + uint64_t current_filtered_rg_start_ = 0; // Absolute row-group start row number + + // Target row groups with row ranges for none page-level filtering and page-level filtering + std::vector target_row_groups_; + + // Track pre-buffered ranges so we can wait on destruction + std::vector<::arrow::io::ReadRange> prebuffered_ranges_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index 12f90a1f..3ac7f62e 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -117,10 +117,11 @@ class FileReaderWrapperTest : public ::testing::Test { ASSERT_OK(format_writer->AddBatch(batch->GetData())); } - Result> PrepareReaderWrapper(const std::string& file_path) { + Result> PrepareReaderWrapper( + const std::string& file_path, int64_t wrapper_batch_size = 0) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in, fs_->Open(file_path)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - auto input_stream = std::make_unique(in, arrow_pool_, file_length); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); + auto input_stream = std::make_unique(in, file_length, arrow_pool_); ::parquet::arrow::FileReaderBuilder file_reader_builder; ::parquet::ReaderProperties reader_properties; reader_properties.enable_buffered_stream(); @@ -136,10 +137,11 @@ class FileReaderWrapperTest : public ::testing::Test { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(arrow_pool_.get()) ->properties(arrow_reader_props) ->Build(&file_reader)); - return FileReaderWrapper::Create(std::move(file_reader)); + return FileReaderWrapper::Create(std::move(file_reader), wrapper_batch_size, arrow_pool_); } - void PrepareParquetFile(const std::string& file_path, int32_t row_count) { + void PrepareParquetFile(const std::string& file_path, int32_t row_count, + bool enable_page_index = false, int32_t write_batch_size = 10) { auto schema_pair = PrepareArrowSchema(); const auto& arrow_schema = schema_pair.first; const auto& struct_type = schema_pair.second; @@ -147,9 +149,14 @@ class FileReaderWrapperTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/false)); ::parquet::WriterProperties::Builder builder; - builder.write_batch_size(10); + builder.write_batch_size(write_batch_size); builder.max_row_group_length(1000); builder.enable_store_decimal_as_integer(); + if (enable_page_index) { + builder.enable_write_page_index(); + builder.disable_dictionary(); + builder.data_pagesize(1); + } auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN( std::shared_ptr format_writer, @@ -190,7 +197,9 @@ TEST_F(FileReaderWrapperTest, EmptyFile) { } TEST_F(FileReaderWrapperTest, NullFileReader) { - ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr), + ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr, + /*batch_size=*/0, + /*pool=*/arrow_pool_), "file reader wrapper create failed. file reader is nullptr"); } @@ -240,6 +249,131 @@ TEST_F(FileReaderWrapperTest, Simple) { ASSERT_EQ(5500, reader_wrapper->GetPreviousBatchFirstRowNumber().value()); } +/// Regression: when batch_size_ is 0 (the default) and a row group is consumed via +/// the page-filtered streaming path, we must not pass 0 to TableBatchReader::set_chunksize +/// — that would make ReadNext spin forever on zero-row batches. The wrapper now +/// translates 0 to int64_max so the reader produces one batch covering all matched rows. +TEST_F(FileReaderWrapperTest, PageFilteredZeroBatchSizeDoesNotHang) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "page_zero_batch.parquet"); + PrepareParquetFile(file_path, /*row_count=*/200, /*enable_page_index=*/true); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + ASSERT_EQ(1, reader_wrapper->GetNumberOfRowGroups()); + + // Inject a per-RG RowRanges to drive the page-filtered streaming path. Two non- + // contiguous ranges keep the test honest about RowRanges semantics; the actual + // numbers don't matter as long as their total falls inside the row group. + RowRanges rr({RowRanges::Range(0, 49), RowRanges::Range(100, 149)}); + + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rr)}, + all_columns)); + int64_t total = 0; + int64_t batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + total += batch->num_rows(); + ++batch_count; + ASSERT_LT(batch_count, 1000) << "Next() did not converge — likely an infinite loop"; + } + ASSERT_EQ(100, total); + ASSERT_GE(batch_count, 1); +} + +/// SeekToRow back to a previously-consumed page-filtered row group must rebuild the +/// per-RG streaming reader from row_group_row_ranges_ and re-yield the same rows. +/// The page-filter path holds no per-RG cache that consumption could destroy; the +/// reader is constructed on demand each time, mirroring Arrow's stateless +/// GetRecordBatchReader for the fully-matched path. +TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "seek_back.parquet"); + // 2000 rows produces 2 row groups (max_row_group_length=1000) with page index enabled. + PrepareParquetFile(file_path, /*row_count=*/2000, /*enable_page_index=*/true); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + ASSERT_EQ(2, reader_wrapper->GetNumberOfRowGroups()); + + // Both RGs page-filtered. RowRanges are RG-local: RG0 keeps 40 rows, RG1 keeps 50. + std::map row_ranges_map; + row_ranges_map[0] = RowRanges(RowRanges::Range(10, 49)); + row_ranges_map[1] = RowRanges(RowRanges::Range(100, 149)); + + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, + /*ranges=*/row_ranges_map[0]), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/true, + /*ranges=*/row_ranges_map[1])}, + all_columns)); + + auto count_all_rows = [&](int64_t* out_total) { + int64_t total = 0; + while (true) { + auto next = reader_wrapper->Next(); + if (!next.ok()) return next.status(); + auto batch = std::move(next).value(); + if (!batch) break; + total += batch->num_rows(); + } + *out_total = total; + return Status::OK(); + }; + + int64_t first_total = 0; + ASSERT_OK(count_all_rows(&first_total)); + ASSERT_EQ(90, first_total); // 40 + 50 + + // Seek back to row 0 (start of RG0). The on-demand reader construction means RG0 + // is read again from scratch, producing the same 90 rows total. + ASSERT_OK(reader_wrapper->SeekToRow(0)); + + int64_t second_total = 0; + ASSERT_OK(count_all_rows(&second_total)); + ASSERT_EQ(90, second_total); +} + +/// When the page-level predicate matches more rows than the wrapper's batch_size, +/// the page-filtered streaming path must split the filtered rows across multiple +/// Next() calls. Pages are written 3 rows wide (write_batch_size=3 with +/// data_pagesize=1) so that filtered rows span multiple page-sized chunks; the +/// emitted batches must (a) sum to the RowRanges row count and (b) never exceed +/// the configured batch_size — TableBatchReader additionally caps each batch at +/// the underlying chunk boundary, which is fine as long as the cap holds. +TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { + constexpr int32_t kRowCount = 60; + constexpr int32_t kPageRowCount = 3; + constexpr int64_t kExpectedTotal = 30; + + std::string file_path = PathUtil::JoinPath(dir_->Str(), "page_split.parquet"); + PrepareParquetFile(file_path, kRowCount, /*enable_page_index=*/true, + /*write_batch_size=*/kPageRowCount); + + // Keep rows [0, 29] — the first 10 pages of the row group. + RowRanges rr({RowRanges::Range(0, kExpectedTotal - 1)}); + + for (int64_t batch_size : {int64_t{1}, int64_t{2}, int64_t{3}, int64_t{5}, int64_t{10}}) { + SCOPED_TRACE("batch_size=" + std::to_string(batch_size)); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, batch_size)); + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rr)}, + {0, 1, 2})); + + int64_t total = 0; + int64_t batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + ASSERT_GT(batch->num_rows(), 0); + ASSERT_LE(batch->num_rows(), batch_size); + total += batch->num_rows(); + ++batch_count; + } + ASSERT_EQ(kExpectedTotal, total); + const int64_t min_batches = (kExpectedTotal + batch_size - 1) / batch_size; + ASSERT_GE(batch_count, min_batches); + } +} + TEST_F(FileReaderWrapperTest, GetRowGroupRanges) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); @@ -253,45 +387,93 @@ TEST_F(FileReaderWrapperTest, GetRowGroupRanges) { ASSERT_TRUE(ranges.empty()); } -TEST_F(FileReaderWrapperTest, ReadRangesToRowGroupIds) { +TEST_F(FileReaderWrapperTest, ApplyReadRanges) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - std::set expected_row_group_ids = {0, 3, 5}; + + // Prepare with a subset of row groups: {0, 1, 2, 4, 5} + std::vector initial_targets = { + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/2, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/4, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/5, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}; + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + + // Apply read ranges that match RG 0, 3, 5. Only 0 and 5 are in initial targets. std::vector> read_ranges = { {0, 1000}, {3000, 4000}, {5000, 5500}}; - ASSERT_OK_AND_ASSIGN(auto row_group_ids, reader_wrapper->ReadRangesToRowGroupIds(read_ranges)); - ASSERT_EQ(expected_row_group_ids, row_group_ids); - std::vector> invalid_ranges = { - {0, 1000}, {3000, 4000}, {5000, 5600}}; - ASSERT_NOK_WITH_MSG(reader_wrapper->ReadRangesToRowGroupIds(invalid_ranges), - "not match with row group range bound"); - ASSERT_OK_AND_ASSIGN(row_group_ids, reader_wrapper->ReadRangesToRowGroupIds({})); - ASSERT_TRUE(row_group_ids.empty()); + ASSERT_OK(reader_wrapper->ApplyReadRanges(read_ranges)); + + // Verify: reading should only produce rows from RG 0 (1000 rows) and RG 5 (500 rows). + int64_t total_rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) { + break; + } + total_rows += batch->num_rows(); + } + ASSERT_EQ(1500, total_rows); + + // Apply empty read ranges should result in no data. + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + ASSERT_OK(reader_wrapper->ApplyReadRanges({})); + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + ASSERT_FALSE(batch); } -TEST_F(FileReaderWrapperTest, FilterRowGroupsByReadRanges) { +TEST_F(FileReaderWrapperTest, ApplyReadRangesWiderSecondCall) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - std::set expected_row_group_ids = {0, 5}; - std::vector> read_ranges = { - {0, 1000}, {3000, 4000}, {5000, 5500}}; - ASSERT_OK_AND_ASSIGN(auto row_group_ids, - reader_wrapper->FilterRowGroupsByReadRanges(read_ranges, {0, 1, 2, 4, 5})); - ASSERT_EQ(expected_row_group_ids, row_group_ids); - ASSERT_OK_AND_ASSIGN(row_group_ids, - reader_wrapper->FilterRowGroupsByReadRanges(read_ranges, {})); - ASSERT_TRUE(row_group_ids.empty()); + // Prepare with row groups: {0, 1, 2, 4, 5} + std::vector initial_targets = { + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/2, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/4, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/5, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}; + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + + // First ApplyReadRanges: narrow to RG 0 only. + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}})); + + // Second ApplyReadRanges: widen to RG 0, 1, 2. Previously excluded RG 1, 2 should restore. + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}, {1000, 2000}, {2000, 3000}})); + + // Verify: reading should produce rows from RG 0 + 1 + 2 = 3000 rows. + int64_t total_rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + total_rows += batch->num_rows(); + } + ASSERT_EQ(3000, total_rows); } TEST_F(FileReaderWrapperTest, PrepareForReading) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{1}, - /*column_indices=*/{0})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}, + /*column_indices=*/{0})); // seek before actual read range ASSERT_OK(reader_wrapper->SeekToRow(0)); ASSERT_EQ(1000, reader_wrapper->GetNextRowToRead()); @@ -311,8 +493,12 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_FALSE(record_batch); // empty column indices - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{0, 1}, - /*column_indices=*/{})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}, + /*column_indices=*/{})); ASSERT_EQ(0, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), reader_wrapper->GetPreviousBatchFirstRowNumber().value()); @@ -321,8 +507,9 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_EQ(0, record_batch->num_columns()); // empty row group indices - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{}, - /*column_indices=*/{0})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{}, + /*column_indices=*/{0})); ASSERT_EQ(5500, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), reader_wrapper->GetPreviousBatchFirstRowNumber().value()); diff --git a/src/paimon/format/parquet/floating_point_predicate_utils.h b/src/paimon/format/parquet/floating_point_predicate_utils.h new file mode 100644 index 00000000..a1fb9a7f --- /dev/null +++ b/src/paimon/format/parquet/floating_point_predicate_utils.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/defs.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/literal.h" + +namespace paimon::parquet { + +class FloatingPointPredicateUtils { + public: + FloatingPointPredicateUtils() = delete; + ~FloatingPointPredicateUtils() = delete; + + static bool IsType(FieldType field_type) { + return field_type == FieldType::FLOAT || field_type == FieldType::DOUBLE; + } + + static bool IsZero(const Literal& literal) { + if (literal.IsNull()) { + return false; + } + if (literal.GetType() == FieldType::FLOAT) { + return literal.GetValue() == 0.0f; + } + if (literal.GetType() == FieldType::DOUBLE) { + return literal.GetValue() == 0.0; + } + return false; + } + + static bool IsNegativeZero(const Literal& literal) { + if (!IsZero(literal)) { + return false; + } + if (literal.GetType() == FieldType::FLOAT) { + return std::signbit(literal.GetValue()); + } + return std::signbit(literal.GetValue()); + } + + static bool IsComparison(Function::Type function_type) { + switch (function_type) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + return true; + default: + return false; + } + } +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp new file mode 100644 index 00000000..6b8f9a26 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/page_filtered_row_group_reader.h" + +#include + +#include "arrow/array.h" +#include "arrow/builder.h" +#include "arrow/chunked_array.h" +#include "arrow/io/caching.h" +#include "arrow/io/interfaces.h" +#include "arrow/table.h" +#include "arrow/util/future.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/arrow_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "parquet/arrow/reader.h" +#include "parquet/arrow/reader_internal.h" +#include "parquet/arrow/schema.h" +#include "parquet/metadata.h" +#include "parquet/schema.h" + +namespace paimon::parquet { + +namespace { + +/// Wraps an arrow::Table + TableBatchReader as a RecordBatchReader so the caller can +/// stream batches while ensuring every returned array offset is zero. The Table is held +/// to keep its ChunkedArrays alive for the inner TableBatchReader. +class TableRecordBatchReader : public arrow::RecordBatchReader { + public: + TableRecordBatchReader(std::shared_ptr table, int64_t chunksize, + std::shared_ptr pool) + : table_(std::move(table)), inner_(*table_), pool_(std::move(pool)) { + inner_.set_chunksize(chunksize); + } + + std::shared_ptr schema() const override { + return table_->schema(); + } + + arrow::Status ReadNext(std::shared_ptr* out) override { + ARROW_RETURN_NOT_OK(inner_.ReadNext(out)); + if (!*out) { + return arrow::Status::OK(); + } + + // Page filtering may produce columns with different chunk boundaries. TableBatchReader + // aligns them by slicing columns, which may leave non-zero child offsets. + Result> normalized_result = + ArrowUtils::NormalizeRecordBatchOffsets(*out, pool_.get()); + if (!normalized_result.ok()) { + return ToArrowStatus(normalized_result.status()); + } + *out = std::move(normalized_result).value(); + return arrow::Status::OK(); + } + + private: + std::shared_ptr table_; + arrow::TableBatchReader inner_; + std::shared_ptr pool_; +}; + +/// A FileColumnIterator that installs a data_page_filter on every PageReader it +/// produces, enabling I/O-level page skipping. The base class handles row group +/// iteration; this subclass only decorates the PageReader returned by NextChunk(). +class PageFilteringColumnIterator : public ::parquet::arrow::FileColumnIterator { + public: + PageFilteringColumnIterator( + int column_index, ::parquet::ParquetFileReader* reader, std::vector row_groups, + std::function data_page_filter) + : FileColumnIterator(column_index, reader, std::move(row_groups)), + data_page_filter_(std::move(data_page_filter)) {} + + std::unique_ptr<::parquet::PageReader> NextChunk() override { + std::unique_ptr<::parquet::PageReader> page_reader = FileColumnIterator::NextChunk(); + if (page_reader && data_page_filter_) { + page_reader->set_data_page_filter(data_page_filter_); + } + return page_reader; + } + + private: + std::function data_page_filter_; +}; + +} // namespace + +std::pair PageFilteredRowGroupReader::GetPageRowRange( + const std::vector<::parquet::PageLocation>& page_locations, int32_t page_idx, + int64_t row_group_row_count) { + int64_t first_row = page_locations[page_idx].first_row_index; + int64_t last_row = (page_idx + 1 < static_cast(page_locations.size())) + ? page_locations[page_idx + 1].first_row_index - 1 + : row_group_row_count - 1; + return {first_row, last_row}; +} + +std::function PageFilteredRowGroupReader::MakePageFilter( + const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count) { + auto page_counter = std::make_shared(0); + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + + return [row_ranges, page_locations, num_pages, row_group_row_count, + page_counter](const ::parquet::DataPageStats& /*stats*/) -> bool { + int32_t page_idx = (*page_counter)++; + if (page_idx >= num_pages) { + return false; + } + auto [first_row, last_row] = GetPageRowRange(page_locations, page_idx, row_group_row_count); + return !row_ranges.IsOverlapping(first_row, last_row); + }; +} + +std::pair PageFilteredRowGroupReader::ComputeCompressedRowRanges( + const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count) { + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + const auto& ranges = original_ranges.GetRanges(); + + RowRanges compressed; + int64_t compressed_offset = 0; + + for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { + auto [page_from, page_to] = GetPageRowRange(page_locations, page_idx, row_group_row_count); + int64_t page_size = page_to - page_from + 1; + + if (!original_ranges.IsOverlapping(page_from, page_to)) { + // Page will be skipped by data_page_filter, not in compressed space + continue; + } + + for (const auto& range : ranges) { + if (range.to < page_from) continue; + if (range.from > page_to) break; + int64_t overlap_from = std::max(range.from, page_from); + int64_t overlap_to = std::min(range.to, page_to); + compressed.Add(RowRanges::Range(compressed_offset + (overlap_from - page_from), + compressed_offset + (overlap_to - page_from))); + } + + compressed_offset += page_size; + } + + return {compressed, compressed_offset}; +} + +Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( + int col_idx, const RowRanges& ranges, int64_t total, + ::parquet::arrow::ColumnReader* column_reader) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->ResetLeaf(col_idx, total)); + int64_t current = 0; + for (const auto& range : ranges.GetRanges()) { + int64_t skip = range.from > current ? range.from - current : 0; + int64_t skipped = column_reader->SkipRecords(col_idx, skip); + if (skipped != skip) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: leaf {} expected to skip {} records but skipped {}", + col_idx, skip, skipped)); + } + int64_t to_read = range.Count(); + int64_t read = column_reader->ReadRecords(col_idx, to_read); + if (read != to_read) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: leaf {} expected to read {} records but read {}", + col_idx, to_read, read)); + } + current = range.to + 1; + } + return Status::OK(); +} + +Status PageFilteredRowGroupReader::WaitForPreBuffer( + int32_t row_group_index, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::ParquetFileReader* parquet_reader) { + std::vector rg_vec = {row_group_index}; + std::vector col_vec(column_indices.begin(), column_indices.end()); + if (!pre_buffered) { + ::arrow::io::IOContext io_ctx(pool.get()); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + } + if (!page_ranges.empty()) { + auto status = parquet_reader->WhenBufferedRanges(page_ranges).status(); + if (!status.ok()) { + ::arrow::io::IOContext io_ctx(pool.get()); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); + } + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); + } + return Status::OK(); +} + +Result> PageFilteredRowGroupReader::ReadFilteredField( + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t field_index, const std::vector& column_indices, + const RowRanges& row_ranges, int64_t row_group_row_count, + ::parquet::arrow::FileReader* arrow_file_reader) { + // Factory: set data_page_filter on every leaf (per-leaf OffsetIndex). + // data_page_filter enables I/O-level page skipping for all leaves. + auto factory = + [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count]( + int col_idx, + ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* { + std::function data_page_filter; + if (rg_page_index_reader) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + if (offset_index) { + data_page_filter = MakePageFilter(row_ranges, offset_index, row_group_row_count); + } + } + return new PageFilteringColumnIterator(col_idx, reader, std::vector{row_group_index}, + std::move(data_page_filter)); + }; + + // Build reader tree with leaf column filtering + std::unique_ptr<::parquet::arrow::ColumnReader> column_reader; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow_file_reader->GetColumn(field_index, column_indices, factory, &column_reader)); + + if (!column_reader) { + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: field {} has no matching leaf columns " + "(row_group={})", + field_index, row_group_index)); + } + + // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each + // leaf column independently + + for (int col_idx : column_reader->LeafColumnIndices()) { + RowRanges effective_ranges = row_ranges; + int64_t effective_total = row_group_row_count; + if (rg_page_index_reader) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + if (offset_index) { + auto [compressed, total] = + ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); + effective_ranges = std::move(compressed); + effective_total = total; + } + } + + PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(col_idx, effective_ranges, effective_total, + column_reader.get())); + } + + // Build the Arrow array (TransferColumnData for leaves + assemble for nested) + std::shared_ptr chunked_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + column_reader->BuildArray(row_ranges.RowCount(), &chunked_array)); + + return chunked_array; +} + +Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( + const TargetRowGroup& target_row_group, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader) { + auto parquet_reader = arrow_file_reader->parquet_reader(); + const auto& row_ranges = target_row_group.GetRowRanges(); + int32_t row_group_index = target_row_group.GetRowGroupIndex(); + + int64_t expected_rows = row_ranges.RowCount(); + + if (!row_ranges.IsEmpty()) { + PAIMON_RETURN_NOT_OK(WaitForPreBuffer(row_group_index, column_indices, cache_options, + pre_buffered, page_ranges, pool, parquet_reader)); + } + + auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); + int64_t row_group_row_count = rg_metadata->num_rows(); + + // reuse RowGroupPageIndexReader for multiple columns in the same row group to avoid redundant + // metadata reads + std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; + auto page_index_reader = parquet_reader->GetPageIndexReader(); + if (page_index_reader) { + rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + } + + const auto& manifest = arrow_file_reader->manifest(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::vector field_indices, + manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end()))); + + std::vector> result_arrays; + result_arrays.reserve(field_indices.size()); + + for (int field_idx : field_indices) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr chunked_array, + ReadFilteredField(rg_page_index_reader, row_group_index, field_idx, column_indices, + row_ranges, row_group_row_count, arrow_file_reader)); + + if (chunked_array->length() != expected_rows) { + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: field {} produced {} rows but expected {} " + "(row_group={})", + field_idx, chunked_array->length(), expected_rows, row_group_index)); + } + + result_arrays.push_back(std::move(chunked_array)); + } + + std::vector> result_fields; + for (size_t i = 0; i < result_arrays.size(); ++i) { + const auto& field = manifest.schema_fields[field_indices[i]].field; + result_fields.push_back(arrow::field(field->name(), result_arrays[i]->type(), + field->nullable(), field->metadata())); + } + auto result_schema = arrow::schema(result_fields); + + auto table = arrow::Table::Make(result_schema, std::move(result_arrays), expected_rows); + return std::make_unique(std::move(table), max_chunksize, pool); +} + +std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( + const TargetRowGroup& target_row_group, const std::vector& column_indices, + ::parquet::ParquetFileReader* parquet_reader) { + int32_t row_group_index = target_row_group.GetRowGroupIndex(); + const auto& row_ranges = target_row_group.GetRowRanges(); + + std::vector<::arrow::io::ReadRange> ranges; + auto file_metadata = parquet_reader->metadata(); + auto rg_metadata = file_metadata->RowGroup(row_group_index); + int64_t row_group_row_count = rg_metadata->num_rows(); + + auto page_index_reader = parquet_reader->GetPageIndexReader(); + std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; + if (page_index_reader) { + rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + } + + for (int32_t col_idx : column_indices) { + auto col_chunk = rg_metadata->ColumnChunk(col_idx); + int64_t data_page_offset = col_chunk->data_page_offset(); + int64_t data_page_compressed_size = col_chunk->total_compressed_size(); + // Dictionary page: always include if present + if (col_chunk->has_dictionary_page()) { + int64_t dict_offset = col_chunk->dictionary_page_offset(); + int64_t dict_size = data_page_offset - dict_offset; + if (dict_size > 0) { + // if dictionary exists, the data page size should be reduced by the dictionary + data_page_compressed_size -= dict_size; + ranges.push_back({dict_offset, dict_size}); + } + } + + int64_t chunk_end = data_page_offset + data_page_compressed_size; + + // Try to get OffsetIndex for page-level ranges + std::shared_ptr<::parquet::OffsetIndex> offset_index; + if (rg_page_index_reader) { + offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + } + + if (!offset_index) { + // No OffsetIndex: fall back to entire column chunk + ranges.push_back({data_page_offset, data_page_compressed_size}); + continue; + } + + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + + for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { + auto [first_row, last_row] = + GetPageRowRange(page_locations, page_idx, row_group_row_count); + + if (!row_ranges.IsOverlapping(first_row, last_row)) { + continue; + } + + int64_t page_offset = page_locations[page_idx].offset; + int64_t page_size = (page_idx + 1 < num_pages) + ? page_locations[page_idx + 1].offset - page_offset + : chunk_end - page_offset; + ranges.push_back({page_offset, page_size}); + } + } + + return ranges; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h new file mode 100644 index 00000000..1122ece1 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/memory_pool.h" +#include "arrow/record_batch.h" +#include "arrow/type.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" +#include "paimon/result.h" +#include "parquet/arrow/reader.h" +#include "parquet/column_reader.h" +#include "parquet/file_reader.h" +#include "parquet/page_index.h" + +namespace paimon::parquet { + +/// Reads a single row group using page-level filtering. +/// Non-matching rows are skipped at the decoding level via RecordReader::SkipRecords, +/// using RowRanges computed from the page index (ColumnIndex + OffsetIndex). +class PageFilteredRowGroupReader { + public: + PageFilteredRowGroupReader() = delete; + ~PageFilteredRowGroupReader() = delete; + + /// Read a row group with page-level filtering. + /// @param target_row_group Target row group with index and row ranges + /// @param column_indices Leaf column indices to read + /// @param pool Memory pool + /// @param cache_options Cache options for PreBuffer + /// @param pre_buffered If true, assumes PreBuffer was already called externally + /// and only waits via WhenBuffered (no redundant PreBuffer). + /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered + /// @param max_chunksize Per-batch row cap for the returned reader. + /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation + /// @return A RecordBatchReader streaming the filtered rows. + static Result> ReadFilteredRowGroup( + const TargetRowGroup& target_row_group, const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader); + + /// Compute the byte ranges of pages that overlap with the given RowRanges. + /// Uses OffsetIndex to determine per-page file offsets and sizes. + /// Includes dictionary pages unconditionally. + /// Falls back to entire column chunk range if OffsetIndex is unavailable. + static std::vector<::arrow::io::ReadRange> ComputePageRanges( + const TargetRowGroup& target_row_group, const std::vector& column_indices, + ::parquet::ParquetFileReader* parquet_reader); + + private: + /// Get the [first_row, last_row] range of a page given page locations. + static std::pair GetPageRowRange( + const std::vector<::parquet::PageLocation>& page_locations, int32_t page_idx, + int64_t row_group_row_count); + + /// Wait for pre-buffered data to become available before reading. + static Status WaitForPreBuffer(int32_t row_group_index, + const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, + bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, + std::shared_ptr<::arrow::MemoryPool> pool, + ::parquet::ParquetFileReader* parquet_reader); + + /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. + static std::function MakePageFilter( + const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count); + + /// Compute compressed RowRanges after data_page_filter skips non-matching pages. + static std::pair ComputeCompressedRowRanges( + const RowRanges& original_ranges, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); + + /// Reset the given leaf and replay the skip/read pattern derived from `ranges` + /// directly against the ColumnReader (ResetLeaf + SkipRecords/ReadRecords). + static Status ExecuteSkipReadPattern(int col_idx, const RowRanges& ranges, int64_t total, + ::parquet::arrow::ColumnReader* column_reader); + + /// Read a field (flat or nested) using ColumnReader tree. + /// Sets data_page_filter on all leaves via factory, then drives each leaf + /// independently via ResetLeaf/SkipRecords/ReadRecords using its own + /// compressed_ranges. + static Result> ReadFilteredField( + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t field_index, const std::vector& column_indices, + const RowRanges& row_ranges, int64_t row_group_row_count, + ::parquet::arrow::FileReader* arrow_file_reader); +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp new file mode 100644 index 00000000..2ccdc4e4 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -0,0 +1,2011 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/page_filtered_row_group_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_nested.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/defs.h" +#include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/roaring_bitmap32.h" +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#include "parquet/properties.h" + +namespace paimon { +class Predicate; +} // namespace paimon + +namespace paimon::parquet::test { + +/// Test fixture for page-level filtering. +/// Creates Parquet files with multiple row groups and small page sizes to ensure +/// multiple pages per row group, enabling page-level filtering tests. +class PageFilteredRowGroupReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + } + + /// Write a Parquet file with controlled page boundaries. + /// @param file_name Output file name + /// @param struct_array Data to write + /// @param write_batch_size Controls page size (number of rows per page) + /// @param max_row_group_length Controls row group size + void WriteTestFile(const std::string& file_name, + const std::shared_ptr& struct_array, + int32_t write_batch_size, int64_t max_row_group_length, + bool enable_dictionary = false, int64_t data_page_size = 1) { + auto data_type = struct_array->struct_type(); + auto data_schema = arrow::schema(data_type->fields()); + auto data_arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*struct_array, data_arrow_array.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_name, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder builder; + builder.write_batch_size(write_batch_size); + builder.max_row_group_length(max_row_group_length); + if (enable_dictionary) { + builder.enable_dictionary(); + } else { + builder.disable_dictionary(); // Ensure page index min/max are meaningful + } + builder.enable_write_page_index(); // Enable page index for page-level filtering + // Data page size controls when a page is flushed. The default of 1 byte forces a new + // page after every write_batch_size rows (each batch becomes one page), giving pages + // aligned across columns. A larger byte-based value combined with write_batch_size=1 + // instead lets columns of different physical widths flush pages at different row + // counts, producing intentionally misaligned pages across leaves. + builder.data_pagesize(data_page_size); + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, data_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(data_arrow_array.get())); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Close()); + } + + /// Read back a Parquet file with an optional predicate and page index filter enabled. + /// Returns the collected result as a ChunkedArray. + void ReadWithPredicateImpl(const std::string& file_name, + const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + std::shared_ptr* out, + int32_t batch_size = 1024) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true"; + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(*out, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + } + + /// Read back a Parquet file with a predicate, a bitmap, and page index filter enabled. + void ReadWithPredicateAndBitmapImpl(const std::string& file_name, + const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + const RoaringBitmap32& bitmap, + std::shared_ptr* out, + const std::map options = {}, + int32_t batch_size = 1024) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + + ASSERT_OK_AND_ASSIGN( + auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, bitmap)); + ASSERT_OK_AND_ASSIGN(*out, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + } + + protected: + std::shared_ptr arrow_pool_; + std::shared_ptr pool_; + std::shared_ptr fs_; + std::unique_ptr dir_; +}; + +// Helper: build a StructArray with N rows of int32 "val" column with sequential values. +// val[i] = i for i in [0, N). +static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder val_builder; + EXPECT_TRUE(val_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + val_builder.UnsafeAppend(i); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + return arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); +} + +// Helper: build a StructArray with two int32 columns: "a" and "b". +// a[i] = i, b[i] = i * 10, for i in [0, N). +static std::shared_ptr MakeTwoColumnData(int32_t num_rows) { + arrow::Int32Builder a_builder, b_builder; + EXPECT_TRUE(a_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(b_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + a_builder.UnsafeAppend(i); + b_builder.UnsafeAppend(i * 10); + } + auto a_array = a_builder.Finish().ValueOrDie(); + auto b_array = b_builder.Finish().ValueOrDie(); + auto field_a = arrow::field("a", arrow::int32()); + auto field_b = arrow::field("b", arrow::int32()); + return arrow::StructArray::Make({a_array, b_array}, {field_a, field_b}).ValueOrDie(); +} + +/// Test: page-level filtering correctly skips non-matching pages. +/// +/// Scenario: 100 rows, 10 rows per page, 1 row group. +/// val[i] = i. Predicate: val >= 50. Pages 0-4 (rows 0-49) should be skipped, +/// pages 5-9 (rows 50-99) should be read. +TEST_F(PageFilteredRowGroupReaderTest, SingleRowGroupPartialPageMatch) { + std::string file_name = dir_->Str() + "/single_rg_partial.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 50-99 = 50 rows + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Verify actual values + auto flat = result->chunk(0); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)) << "Mismatch at index " << i; + } +} + +/// Test: predicate matches all pages → same as unfiltered read. +TEST_F(PageFilteredRowGroupReaderTest, AllPagesMatch) { + std::string file_name = dir_->Str() + "/all_match.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); +} + +/// Test: predicate matches no pages → empty result. +TEST_F(PageFilteredRowGroupReaderTest, NoPagesMatch) { + std::string file_name = dir_->Str() + "/no_match.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(999)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + // No matching rows; result should be null (empty) + ASSERT_FALSE(result); +} + +/// Test: multiple row groups, page filtering active on some. +/// +/// 200 rows, 10 rows per page, 50 rows per row group → 4 row groups. +/// Predicate: val >= 150. Row groups 0-2 (rows 0-149) should be eliminated entirely. +/// Row group 3 (rows 150-199): all pages match → full read, no page filtering. +TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsFullElimination) { + std::string file_name = dir_->Str() + "/multi_rg_elim.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(150)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Verify values are 150-199 + auto flat = result->chunk(0); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(150 + i, val_arr->Value(i)); + } +} + +/// Test: multiple row groups, partial page match within a row group. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Predicate: val >= 50 AND val < 150. +/// Row group 0 (rows 0-99): pages 0-4 skipped, pages 5-9 read → 50 rows +/// Row group 1 (rows 100-199): pages 0-4 read, pages 5-9 skipped → 50 rows +/// Total: 100 rows +TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsPartialPageMatch) { + std::string file_name = dir_->Str() + "/multi_rg_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(50)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(150))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + // Collect all values and verify they are 50-149 + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(50 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset; + ++offset; + } + } + ASSERT_EQ(100, offset); +} + +/// Test: two columns remain aligned after page-level filtering. +/// +/// 100 rows, a[i] = i, b[i] = i*10. 10 rows per page. +/// Predicate on "a": a >= 50. After filtering, b should be b[50..99] = {500, 510, ..., 990}. +TEST_F(PageFilteredRowGroupReaderTest, MultiColumnAlignment) { + std::string file_name = dir_->Str() + "/multi_col.parquet"; + auto data = MakeTwoColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = + arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"a", FieldType::INT, Literal(50)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(struct_arr); + auto a_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + auto b_arr = std::dynamic_pointer_cast(struct_arr->field(1)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, a_arr->Value(i)); + ASSERT_EQ((50 + i) * 10, b_arr->Value(i)); + } +} + +/// Test: predicate matches pages in the middle of a row group. +/// +/// 100 rows, 10 rows per page. Predicate: val >= 30 AND val < 70. +/// Pages 0-2 (rows 0-29) skipped, pages 3-6 (rows 30-69) read, pages 7-9 (rows 70-99) skipped. +TEST_F(PageFilteredRowGroupReaderTest, MiddlePagesMatch) { + std::string file_name = dir_->Str() + "/middle_pages.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(30)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(70))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(30 + offset, val_arr->Value(j)); + ++offset; + } + } + ASSERT_EQ(40, offset); +} + +/// Test: no predicate → all data returned (no filtering). +TEST_F(PageFilteredRowGroupReaderTest, NoPredicate) { + std::string file_name = dir_->Str() + "/no_predicate.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, /*predicate=*/nullptr, &result); + ASSERT_NE(nullptr, result); + ASSERT_EQ(100, result->length()); +} + +/// Test: page filtering with EQUAL predicate that matches a single page. +/// +/// 100 rows, 10 rows per page. Predicate: val == 55. +/// Only page 5 (rows 50-59) should match, containing value 55. +TEST_F(PageFilteredRowGroupReaderTest, EqualPredicateSinglePageMatch) { + std::string file_name = dir_->Str() + "/equal_single_page.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Page 5 has rows 50-59, which includes 55. The entire page is returned. + ASSERT_EQ(10, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } +} + +/// Test: page filtering with LessThan predicate. +/// +/// 100 rows, 10 rows per page. Predicate: val < 25. +/// Pages 0-2 (rows 0-29) match (page 2 has min=20 < 25). +/// Pages 3-9 don't match. +TEST_F(PageFilteredRowGroupReaderTest, LessThanPredicatePageMatch) { + std::string file_name = dir_->Str() + "/less_than.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(25)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Pages 0 (0-9), 1 (10-19), 2 (20-29) match because their min < 25. + // Page 2 has min=20, max=29, and 20 < 25, so it matches. + ASSERT_EQ(30, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } +} + +/// Test: large data with multiple row groups and page filtering. +/// +/// 1000 rows, 10 rows per page, 200 rows per row group → 5 row groups. +/// Predicate: val >= 500 AND val < 700. +/// Row groups 0,1 (rows 0-399): all pages eliminated +/// Row group 2 (rows 400-599): pages 0-9 (400-499) eliminated, pages 10-19 (500-599) read +/// Row group 3 (rows 600-799): pages 0-9 (600-699) read, pages 10-19 (700-799) eliminated +/// Row group 4 (rows 800-999): all pages eliminated +/// Total: 200 rows (500-699) +TEST_F(PageFilteredRowGroupReaderTest, LargeDataMultiRowGroupPageFilter) { + std::string file_name = dir_->Str() + "/large_data.parquet"; + auto data = MakeSequentialIntData(1000); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/200); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(500)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(700))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(200, result->length()); + + // Verify values are 500-699 + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(500 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset; + ++offset; + } + } + ASSERT_EQ(200, offset); +} + +/// Test: string column page filtering. +/// +/// Write 40 rows with string values: "aaa_00", "aaa_01", ..., "aaa_09", +/// "bbb_10", ..., "bbb_19", "ccc_20", ..., "ccc_29", "ddd_30", ..., "ddd_39". +/// 10 rows per page → 4 pages. Predicate: val >= "ccc" should match pages 2-3. +TEST_F(PageFilteredRowGroupReaderTest, StringColumnPageFilter) { + std::string file_name = dir_->Str() + "/string_filter.parquet"; + + arrow::StringBuilder str_builder; + ASSERT_TRUE(str_builder.Reserve(40).ok()); + std::vector prefixes = {"aaa", "bbb", "ccc", "ddd"}; + for (int32_t i = 0; i < 40; ++i) { + std::string val = prefixes[i / 10] + "_" + (i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(str_builder.Append(val).ok()); + } + auto str_array = str_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::utf8()); + auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie(); + + WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/40); + + auto read_schema = arrow::schema({field}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::STRING, + Literal(FieldType::STRING, "ccc", 3)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Pages 2 (ccc_20..ccc_29) and 3 (ddd_30..ddd_39) should match. + ASSERT_EQ(20, result->length()); +} + +/// Test: ComputePageRanges returns only matching page byte ranges. +/// +/// 100 rows, 10 rows per page, 1 row group with page index enabled. +/// RowRanges = [50, 59] (page 5 only). Should return exactly 1 page range per column. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_partial.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Open as raw ParquetFileReader + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader); + + // Single page match: rows [50, 59] = page 5 + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(50, 59)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + /*column_indices=*/{0}, parquet_reader.get()); + + // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled) + ASSERT_EQ(1, ranges.size()); + ASSERT_GT(ranges[0].offset, 0); + ASSERT_GT(ranges[0].length, 0); +} + +/// Test: ComputePageRanges returns all page ranges when RowRanges covers entire row group. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_all.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + // All rows match + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(0, 99)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); + + // 10 pages, all matching + ASSERT_EQ(10, ranges.size()); + for (const auto& r : ranges) { + ASSERT_GT(r.offset, 0); + ASSERT_GT(r.length, 0); + } +} + +/// Test: ComputePageRanges returns no page ranges for empty RowRanges. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_none.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + RowRanges row_ranges; // empty + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); + + ASSERT_EQ(0, ranges.size()); +} + +/// Test: ComputePageRanges with multiple columns returns ranges for each column. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { + std::string file_name = dir_->Str() + "/compute_ranges_multi_col.parquet"; + auto data = MakeTwoColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + // Match page 5 only (rows 50-59) + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(50, 59)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + {0, 1}, parquet_reader.get()); + + // 1 matching page per column = 2 ranges total + ASSERT_EQ(2, ranges.size()); + // Ranges should be at different offsets (different columns) + ASSERT_NE(ranges[0].offset, ranges[1].offset); +} + +/// Test: ComputePageRanges with multiple matching pages. +/// +/// 100 rows, 10 per page. RowRanges = [20,29] + [70,79] = pages 2 and 7. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { + std::string file_name = dir_->Str() + "/compute_ranges_multi_page.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(20, 29)); + row_ranges.Add(RowRanges::Range(70, 79)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}, + parquet_reader.get()); + + // 2 matching pages for 1 column + ASSERT_EQ(2, ranges.size()); + // Pages should be at increasing offsets + ASSERT_LT(ranges[0].offset, ranges[1].offset); +} + +/// Test: variable-length columns are streamed across multiple offset-normalized +/// RecordBatches when batch_size is smaller than the matched row count, instead of +/// being concatenated into a single RecordBatch via CombineChunks. +/// +/// This verifies the alignment with Arrow's standard TableBatchReader path: +/// multi-chunk binary/string columns split along chunk + batch_size boundaries. It +/// asserts correctness and the multi-batch shape. +TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) { + std::string file_name = dir_->Str() + "/string_multi_batch.parquet"; + + arrow::StringBuilder str_builder; + ASSERT_TRUE(str_builder.Reserve(60).ok()); + // 6 pages of 10 rows each: prefix "p0_".."p5_" so each page has a distinct min/max. + for (int32_t i = 0; i < 60; ++i) { + std::string val = + "p" + std::to_string(i / 10) + "_" + (i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(str_builder.Append(val).ok()); + } + auto str_array = str_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::utf8()); + auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie(); + + WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/60); + + // Predicate matches pages 2..5 (40 rows: "p2_20".."p5_59"). batch_size=7 forces + // the wrapper to surface multiple batches per page-filtered RG. + auto read_schema = arrow::schema({field}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::STRING, + Literal(FieldType::STRING, "p2", 2)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + // Multi-batch shape: with 40 matched rows and batch_size=7 we expect at least + // ceil(40/7)=6 chunks. Anything > 1 already proves we did not collapse to a single + // post-CombineChunks RecordBatch. + ASSERT_GT(result->num_chunks(), 1); + + // Content correctness: rows arrive in the original page order, "p2_20" through "p5_59". + int64_t seen = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_chunk = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_chunk); + auto str_chunk = std::dynamic_pointer_cast(struct_chunk->field(0)); + ASSERT_TRUE(str_chunk); + for (int64_t j = 0; j < str_chunk->length(); ++j) { + int32_t row = 20 + static_cast(seen); + std::string expected = + "p" + std::to_string(row / 10) + "_" + (row < 10 ? "0" : "") + std::to_string(row); + ASSERT_EQ(expected, str_chunk->GetString(j)); + ++seen; + } + } + ASSERT_EQ(40, seen); +} + +TEST_F(PageFilteredRowGroupReaderTest, NormalizesSlicedBatchOffsets) { + std::string file_name = dir_->Str() + "/normalized_sliced_offsets.parquet"; + std::shared_ptr data = MakeSequentialIntData(60); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/60); + + std::shared_ptr read_schema = + arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(20)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + ASSERT_GT(result->num_chunks(), 1); +} + +/// Test: end-to-end page-filtered read produces correct results when using page-level PreBuffer. +/// +/// This exercises the full path: ComputePageRanges → PreBufferRanges → CachedInputStream → +/// ReadFilteredRowGroup with page_ranges. +TEST_F(PageFilteredRowGroupReaderTest, EndToEndPageLevelPreBuffer) { + std::string file_name = dir_->Str() + "/e2e_page_prebuffer.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Read via the standard ParquetFileBatchReader path (page index enabled) + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55)); + + // Use small batch_size to verify batched consumption of page-filtered results + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/3); + ASSERT_TRUE(result); + // Page 5 (rows 50-59) matches, should return 10 rows + ASSERT_EQ(10, result->length()); + + // Verify actual values across chunks + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(50 + offset, val_arr->Value(j)); + ++offset; + } + } + ASSERT_EQ(10, offset); +} + +/// Test: ComputePageRanges with dictionary encoding produces correct chunk_end. +/// +/// When dictionary encoding is enabled, the column chunk layout is: +/// [Dictionary Page] [Data Page 0] [Data Page 1] ... [Data Page N] +/// And total_compressed_size covers the entire chunk starting from dictionary_page_offset. +/// +/// The bug: chunk_end = data_page_offset + total_compressed_size is wrong because +/// total_compressed_size already includes the dictionary page size. The correct +/// chunk_end should be dictionary_page_offset + total_compressed_size. +/// +/// This test verifies that: +/// 1. No range exceeds the true chunk boundary (overshoot regression). +/// 2. At least one non-dictionary data-page range is present (not truncated). +/// 3. The maximum range_end equals true_chunk_end when requesting all rows. +/// 4. End-to-end reads with page-level filtering return correct query results. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) { + std::string file_name = dir_->Str() + "/compute_ranges_dict.parquet"; + + // Use low-cardinality data to ensure dictionary encoding is actually used. + // 100 rows with values cycling through 0..9 → dictionary will have 10 entries. + arrow::Int32Builder val_builder; + ASSERT_TRUE(val_builder.Reserve(100).ok()); + for (int32_t i = 0; i < 100; ++i) { + val_builder.UnsafeAppend(i % 10); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + auto struct_array = arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); + + // Write with dictionary encoding enabled and 1 row per page. + // Each page has min==max==val for that row, enabling precise page-level skipping. + WriteTestFile(file_name, struct_array, /*write_batch_size=*/1, + /*max_row_group_length=*/100, /*enable_dictionary=*/true); + + // Open the file and verify metadata confirms dictionary page presence + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader); + + auto file_metadata = parquet_reader->metadata(); + auto rg_metadata = file_metadata->RowGroup(0); + auto col_chunk = rg_metadata->ColumnChunk(0); + + // Precondition: dictionary page must exist for this test to be meaningful + ASSERT_TRUE(col_chunk->has_dictionary_page()); + + int64_t dict_offset = col_chunk->dictionary_page_offset(); + int64_t data_page_offset = col_chunk->data_page_offset(); + int64_t total_compressed_size = col_chunk->total_compressed_size(); + + // The true chunk end is dict_offset + total_compressed_size + int64_t true_chunk_end = dict_offset + total_compressed_size; + // The buggy chunk end would be data_page_offset + total_compressed_size + int64_t buggy_chunk_end = data_page_offset + total_compressed_size; + + // Sanity: dict page is before data pages, so buggy end > true end + ASSERT_LT(dict_offset, data_page_offset); + ASSERT_GT(buggy_chunk_end, true_chunk_end); + // Now call ComputePageRanges with all rows matching + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(0, 99)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges), + /*column_indices=*/{0}, parquet_reader.get()); + + ASSERT_FALSE(ranges.empty()); + + // --- Check 1: No range should extend beyond the true chunk end --- + // With the bug, the last data page's range would use chunk_end = data_page_offset + + // total_compressed_size, which overshoots by the dictionary page size. + for (auto& range : ranges) { + int64_t range_end = range.offset + range.length; + ASSERT_LE(range_end, true_chunk_end); + } + + // --- Check 2: At least one non-dictionary data-page range is present --- + // Guards against truncation: if only the dictionary range is returned, the test + // would still pass the overshoot check but miss that data pages are lost. + int data_page_range_count = 0; + for (const auto& range : ranges) { + if (range.offset >= data_page_offset) { + ++data_page_range_count; + } + } + ASSERT_GE(data_page_range_count, 1); + + // --- Check 3: Maximum range_end equals true_chunk_end when requesting all rows --- + int64_t max_range_end = 0; + for (const auto& range : ranges) { + int64_t range_end = range.offset + range.length; + max_range_end = std::max(max_range_end, range_end); + } + ASSERT_EQ(max_range_end, true_chunk_end); + + // --- Check 4: No range exceeds file size --- + for (const auto& range : ranges) { + ASSERT_LE(range.offset + range.length, static_cast(length)); + } + + // --- End-to-end check 1: read all rows (no predicate filtering) --- + // Verifies that reading a dictionary-encoded file with page index enabled + // returns all 100 rows with correct values. + auto read_schema = arrow::schema({field}); + auto predicate_all = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0)); + std::shared_ptr result_all; + ReadWithPredicateImpl(file_name, read_schema, predicate_all, &result_all); + ASSERT_TRUE(result_all); + ASSERT_EQ(100, result_all->length()); + + // --- End-to-end check 2: full range query with page level skipping --- + // Build expected array: val = i % 10 for i in [0, 100), wrapped in a struct. + // Concatenate all chunks and compare with Equals + auto actual_struct_arr = arrow::Concatenate(result_all->chunks()).ValueOrDie(); + ASSERT_TRUE(actual_struct_arr->Equals(struct_array)); + + // --- End-to-end check 3: partial-row query with page-level skipping --- + // Predicate val >= 7 skips pages where val < 7, keeping only val in {7,8,9}. + // Out of 100 rows, 30 rows satisfy val >= 7 (3 per cycle × 10 cycles). + auto predicate_partial = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(7)); + std::shared_ptr result_partial; + ReadWithPredicateImpl(file_name, read_schema, predicate_partial, &result_partial); + ASSERT_TRUE(result_partial); + + // Build expected StructArray and compare with Equals + arrow::Int32Builder expected_builder; + ASSERT_TRUE(expected_builder.Reserve(30).ok()); + for (int32_t i = 0; i < 100; ++i) { + if (i % 10 >= 7) { + expected_builder.UnsafeAppend(i % 10); + } + } + auto expected_array = expected_builder.Finish().ValueOrDie(); + auto expected_struct = arrow::StructArray::Make({expected_array}, {field}).ValueOrDie(); + auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie(); + ASSERT_TRUE(partial_concat->Equals(expected_struct)); +} +/// Helper: build an Int32Array with sequential values 0..N-1. +static std::shared_ptr MakeIdColumn(int32_t num_rows) { + arrow::Int32Builder id_builder; + EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + id_builder.UnsafeAppend(i); + } + return id_builder.Finish().ValueOrDie(); +} + +/// Helper: build a struct array (without id column). +/// x[i] = i * 100, y[i] = i * 100 + 1, for i in [0, N). +static std::shared_ptr MakeNestedStructData(int32_t num_rows) { + arrow::Int32Builder x_builder, y_builder; + EXPECT_TRUE(x_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(y_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + x_builder.UnsafeAppend(i * 100); + y_builder.UnsafeAppend(i * 100 + 1); + } + auto x_array = x_builder.Finish().ValueOrDie(); + auto y_array = y_builder.Finish().ValueOrDie(); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + return arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); +} + +/// Test: rowgroup-level filtering on a file with nested struct columns. +/// +/// Schema: { id: int32, info: struct } +/// Parquet leaf columns: [id=0, info.x=1, info.y=2] +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → page 0-7 skipped, paged 8-9 read → 30 rows expected. +/// The read schema requests both "id" and "info" columns. +TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnPageFilter) { + std::string file_name = dir_->Str() + "/nested_struct_filter.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_info}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 70-99 = 30 rows + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Build expected result: rows 50-99 from the original data + auto expected = data->Slice(70, 30); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: Page-level filtering reading only the predicate column (no nested column in read schema). +/// +/// This verifies that when reading only the "id" column (without the nested struct) +/// +/// Schema: { id: int32, info: struct } +/// Read schema: { id: int32 } +/// Predicate on "id": id >= 70. Page-level filtering active → rows 70-99 (30 rows). +TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { + std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + // Read "id" column only + auto read_schema = arrow::schema({field_id}); + + // Predicate is on "id" (field_index=0 in file schema) + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 70-99 = 30 rows + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + auto result_struct = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(result_struct); + ASSERT_TRUE(data->field(0)->Slice(70, 30)->Equals(result_struct->field(0))); +} + +/// Helper: build a list array (without id column). +/// tags[i] = [i*10, i*10+1], for i in [0, N). +static std::shared_ptr MakeListColumnData(int32_t num_rows) { + auto value_builder = std::make_shared(); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); + for (int32_t i = 0; i < num_rows; ++i) { + EXPECT_TRUE(list_builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(i * 10).ok()); + EXPECT_TRUE(value_builder->Append(i * 10 + 1).ok()); + } + return list_builder.Finish().ValueOrDie(); +} + +/// Helper: build a map array (without id column). +/// props[i] = {"k_i": i * 100}, for i in [0, N). +static std::shared_ptr MakeMapColumnData(int32_t num_rows) { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + for (int32_t i = 0; i < num_rows; ++i) { + EXPECT_TRUE(map_builder.Append().ok()); + std::string key = "k_" + std::to_string(i); + EXPECT_TRUE(key_builder->Append(key).ok()); + EXPECT_TRUE(value_builder->Append(i * 100).ok()); + } + return map_builder.Finish().ValueOrDie(); +} + +/// Test: rowgroup-level filtering on a file with a list column. +/// +/// Schema: { id: int32, tags: list } +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnPageFilter) { + std::string file_name = dir_->Str() + "/nested_list_filter.parquet"; + + auto field_id = arrow::field("id", arrow::int32()); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + + auto id_array = MakeIdColumn(100); + auto tags_array = MakeListColumnData(100); + auto data = + arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_tags}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Build expected result: rows 70-99 from the original data + auto expected = data->Slice(70, 30); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: rowgroup filtering on a file with a map column. +/// +/// Schema: { id: int32, props: map } +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnPageFilter) { + std::string file_name = dir_->Str() + "/nested_map_filter.parquet"; + + auto field_id = arrow::field("id", arrow::int32()); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + + auto id_array = MakeIdColumn(100); + auto props_array = MakeMapColumnData(100); + auto data = + arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_props}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Build expected result: rows 70-99 from the original data + auto expected = data->Slice(70, 30); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: rowgroup-level filtering with multiple adjacent nested columns (struct + list). +/// +/// Schema: { id: int32, info: struct, tags: list } +/// This tests the boundary handling when two nested fields are adjacent in the schema. +/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { + std::string file_name = dir_->Str() + "/multi_nested.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto tags_array = MakeListColumnData(100); + auto data = arrow::StructArray::Make({id_array, info_array, tags_array}, + {field_id, field_info, field_tags}) + .ValueOrDie(); + + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_info, field_tags}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Build expected result: rows 70-99 from the original data + auto expected = data->Slice(70, 30); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} +/// Test: bitmap hits all pages of a subset of row groups (no predicate). +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// RG0: rows 0-99, RG1: rows 100-199. +/// Bitmap: {0..99} hits all pages of RG0, RG1 is excluded entirely. +/// Expected: 100 rows (0-99). +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesSomeRowGroups) { + std::string file_name = dir_->Str() + "/bitmap_all_pages_rg.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // hits all of RG0 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } +} + +/// Test: bitmap hits partial pages of a row group (no predicate). +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109). +/// Expected: 20 rows (90-109). +TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesAcrossRowGroups) { + std::string file_name = dir_->Str() + "/bitmap_partial_pages_rg.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + // {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109). + bitmap.AddRange(90, 110); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(20, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_EQ(90 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap hits all pages of some row groups and partial pages of others. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} hits all of RG0 + {120..149} hits pages 2-4 of RG1. +/// Expected: 100 (RG0) + 30 (RG1 partial) = 130 rows. +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllAndPartialPagesMixed) { + std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // all of RG0 + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(130, result->length()); + + // Verify: rows 0-99 + 120-149 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(120 + i, val_arr->Value(100 + i)); + } +} + +/// Test: bitmap hits partial pages of a row group, with page-filtered option disabled. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {120..149} hits pages 2-4 of RG1. +/// Expected: 100 rows (100-199) because page-filtered option is disabled, so page-level bitmap is +/// ignored. +TEST_F(PageFilteredRowGroupReaderTest, BitmapWithPageFilteredOptionDisabled) { + std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "false"; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + // Verify: 100-199 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(100 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits all pages of some row groups. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} hits all of RG0. +/// Predicate: val >= 50. Page-level filtering on RG0: pages 5-9. +/// Expected: 50 rows (50-99). +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_all_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // hits all of RG0 + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits partial pages of a row group. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {30..59} hits pages 3-5 of RG0 (rows 30-59). +/// Predicate: val >= 40. Page-level filtering further narrows to pages 4-5 (rows 40-59). +/// Expected: 20 rows (40-59). +TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_partial_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(30, 60); // hits pages 3-5 of RG0 + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(40)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(20, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_EQ(40 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits all pages of some RG and +/// partial pages of another. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} (all of RG0) + {120..149} (pages 2-4 of RG1). +/// Predicate: val >= 50 AND val < 160. +/// RG0: all pages → page-filtered to val>=50 → rows 50-99 (50 rows) +/// RG1: pages 2-4 (120-149) → page-filtered to val>=50 AND val<160 → all match (30 rows) +/// Expected: 80 rows (50-99 + 120-149). +TEST_F(PageFilteredRowGroupReaderTest, BitmapMixedWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_mixed_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // all of RG0 + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(50)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(160))})); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(80, result->length()); + + // Verify: rows 50-99 + 120-149 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(120 + i, val_arr->Value(50 + i)); + } +} + +/// Test: coalesce strategy with default hole_size_limit (32). +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: [0,10), [45, 50), [60, 70). +/// - [0,10) and [45,50) are both in RG0 page 0 ([0,49]); gap = 35 > 32, NOT merged. +/// - [45,50) and [60,70) straddle RG0 page 0 ([0,49]) and page 1 ([50,99]); gap = 10 <= 32, +/// merged across the page boundary, so rows 50-59 are read even though not in the bitmap. +/// Expected: 35 rows ([0,10) + [45, 70)). +TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceTest) { + std::string file_name = dir_->Str() + "/scattered_bitmap.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(35, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + // [0, 9] — not merged (gap to next range = 35 > 32) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); + } + // [45, 69] — merged across page boundary (gap = 10 <= 32) + for (int32_t i = 0; i < 25; ++i) { + ASSERT_EQ(45 + i, val_arr->Value(10 + i)); + } +} + +/// Test: coalesce strategy with hole_size_limit=5 (instead of default 32). +/// +/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70). +/// Both gaps (35 and 10) exceed 5, so no ranges are merged. +/// Expected: 25 rows (exact bitmap selection, no holes filled). +/// Compare with BitmapCoalesceTest which gets 35 rows (gap of 10 is filled with default limit=32). +TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceSmallHoleSizeTest) { + std::string file_name = dir_->Str() + "/coalesce_small_hole.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "5"; + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(25, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + // [0, 9] — not merged (gap = 35 > 5) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); + } + // [45, 49] — not merged (gap = 10 > 5) + for (int32_t i = 0; i < 5; ++i) { + ASSERT_EQ(45 + i, val_arr->Value(10 + i)); + } + // [60, 69] — not merged + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, val_arr->Value(15 + i)); + } +} + +/// Test: trim strategy for bitmap row filtering. +/// +/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70). +/// Trim produces one range per page (trimmed to first/last selected row in that page): +/// - RG0 page 0 ([0,49]): first selected = 0, last selected = 49 → [0, 49] (50 rows, includes +/// the 35-row gap 10-44 that coalesce keeps as a hole because gap = 35 > 32). +/// - RG0 page 1 ([50,99]): first selected = 60, last selected = 69 → [60, 69] (10 rows; rows +/// 50-59 are NOT read, unlike coalesce which merges across the page boundary). +/// Expected: 60 rows ([0, 50) + [60, 70)). +/// Compare with BitmapCoalesceTest which gets 35 rows. +TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimStrategyTest) { + std::string file_name = dir_->Str() + "/trim_strategy.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim"; + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(60, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + // RG0 page 0 trimmed to [0, 49] — includes gap 10-44 that coalesce skips + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); + } + // RG0 page 1 trimmed to [60, 69] — rows 50-59 not read (unlike coalesce) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, val_arr->Value(50 + i)); + } +} + +/// Test: invalid strategy value returns Status::Invalid. +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapCoalesceTest. +/// Strategy: "invalid" (not one of "coalesce", "trim", "none"). +/// Expected: SetReadSchema returns Status::Invalid. +TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) { + std::string file_name = dir_->Str() + "/invalid_strategy.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "invalid"; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, 1024, nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + auto status = batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, bitmap); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.IsInvalid()); +} + +/// Test: trim strategy with multiple columns — intersection of per-column trimmed ranges. +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Two columns: a[i] = i, b[i] = i * 10. +/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapTrimStrategyTest. +/// Both columns share the same page boundaries, so their trimmed ranges are identical. +/// The intersection is the same as either column alone: +/// - RG0 page 0 ([0,49]): trimmed to [0, 49] (50 rows) +/// - RG0 page 1 ([50,99]): trimmed to [60, 69] (10 rows) +/// Expected: 60 rows. Both columns must remain aligned after trimming. +TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimMultiColumnTest) { + std::string file_name = dir_->Str() + "/trim_multi_col.parquet"; + auto data = MakeTwoColumnData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim"; + + auto read_schema = + arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(60, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto a_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + auto b_arr = std::dynamic_pointer_cast(struct_arr->field(1)); + ASSERT_TRUE(a_arr); + ASSERT_TRUE(b_arr); + // RG0 page 0 trimmed to [0, 49] + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(i, a_arr->Value(i)); + ASSERT_EQ(i * 10, b_arr->Value(i)); + } + // RG0 page 1 trimmed to [60, 69] + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, a_arr->Value(50 + i)); + ASSERT_EQ((60 + i) * 10, b_arr->Value(50 + i)); + } +} + +/// Test: predicate pushdown with all nested column types (struct, list, map). +/// +/// Schema: { id: int32, info: struct, +/// tags: list, props: map } +/// 100 rows, 10 rows per page, 50 rows per row group → 2 row groups. +/// Predicate: id in [15, 29] or id in [80, 99] (Between is inclusive). +/// Read schema: full schema (all columns). +/// Page-level filtering (10 rows/page): +/// Between(15, 29) → pages 1-2 (rows 10-29) +/// Between(80, 99) → pages 8-9 (rows 80-99) +/// Total: 40 rows. +TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) { + std::string file_name = dir_->Str() + "/multi_nested_columns.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + + // Build data with all nested column types using shared helpers + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto tags_array = MakeListColumnData(100); + auto props_array = MakeMapColumnData(100); + auto data = arrow::StructArray::Make({id_array, info_array, tags_array, props_array}, + {field_id, field_info, field_tags, field_props}) + .ValueOrDie(); + + // Write: 10 rows per page, 50 rows per row group → 2 row groups + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + // Read full schema + auto read_schema = arrow::schema({field_id, field_info, field_tags, field_props}); + + // predicate: id in [15, 29] or id in [80, 99] + ASSERT_OK_AND_ASSIGN( + auto predicate, PredicateBuilder::Or( + {PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(15), Literal(29)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(80), Literal(99))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, + /*batch_size=*/1024); + + // Page-level filtering (10 rows/page): + // Between(15, 29) → pages 1-2 (rows 10-29) + // Between(80, 99) → pages 8-9 (rows 80-99) + // Total: 40 rows + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + auto expected = + arrow::ChunkedArray::Make({data->Slice(10, 20), data->Slice(80, 20)}).ValueOrDie(); + ASSERT_TRUE(result->Equals(expected)); +} + +/// Test: sub-column projection of a struct type with page-level filtering. +/// +/// Schema: { id: int32, info: struct } +/// Read schema: { info: struct } — project only x, not y. +/// Predicate: id >= 70 → 30 rows expected. +/// Verifies that reading a sub-column of a nested struct works correctly +/// with page-level filtering and the ColumnReader tree (GetColumn + filter_leaves). +TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) { + std::string file_name = dir_->Str() + "/nested_struct_subcol.parquet"; + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + + auto id_array = MakeIdColumn(100); + auto info_array = MakeNestedStructData(100); + auto data = + arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie(); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + // Read only info.x (sub-column projection: only x, not y) + auto read_schema = arrow::schema({arrow::field("info", arrow::struct_({field_x}))}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + // Result is struct> + auto result_struct = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(result_struct); + ASSERT_EQ(1, result_struct->num_fields()); + + auto info_result = std::dynamic_pointer_cast(result_struct->field(0)); + ASSERT_TRUE(info_result); + ASSERT_EQ(1, info_result->num_fields()); + + auto x_arr = std::dynamic_pointer_cast(info_result->field(0)); + ASSERT_TRUE(x_arr); + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ((70 + i) * 100, x_arr->Value(i)) << "Mismatch at index " << i; + } +} + +/// Helper: build a struct with a flat key plus several nested columns of different +/// physical widths / repetition, so that with a byte-based data page size their +/// leaves flush pages at different row counts (misaligned pages): +/// key: int64 (fixed 8B -> ~5 rows/page) +/// s: struct (x ~10 rows/page, y ~5 rows/page) +/// tags: list (2 values/row -> ~5 rows/page) +/// props: map (variable-width utf8 key -> irregular rows/page) +/// key/s.x/s.y encode the row index (= i); tags/props reuse the shared list/map +/// helpers. Correctness is verified per row by deep-comparing against this array. +static std::shared_ptr MakeMisalignedNestedData(int32_t num_rows) { + arrow::Int64Builder key_builder; + arrow::Int32Builder x_builder; + arrow::Int64Builder y_builder; + EXPECT_TRUE(key_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(x_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(y_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + key_builder.UnsafeAppend(i); + x_builder.UnsafeAppend(i); + y_builder.UnsafeAppend(i); + } + auto key_array = key_builder.Finish().ValueOrDie(); + auto x_array = x_builder.Finish().ValueOrDie(); + auto y_array = y_builder.Finish().ValueOrDie(); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int64()); + auto s_array = arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); + + // Repeated nested leaves (list and map) paginate on value + // bytes, so they misalign with the struct's fixed-width leaves too. + auto tags_array = MakeListColumnData(num_rows); + auto props_array = MakeMapColumnData(num_rows); + + auto field_key = arrow::field("key", arrow::int64()); + auto field_s = arrow::field("s", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + return arrow::StructArray::Make({key_array, s_array, tags_array, props_array}, + {field_key, field_s, field_tags, field_props}) + .ValueOrDie(); +} + +/// Test: page-level filtering across multiple nested columns whose leaf pages are +/// MISALIGNED, within a SINGLE row group. The file mixes a flat key, a +/// struct, a list and a map; with write_batch_size=1 +/// and a byte-based data page size every leaf flushes pages at a different (and, for +/// the utf8 map key, irregular) row count. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesSingleRowGroup) { + std::string file_name = dir_->Str() + "/nested_misaligned_single_rg.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedData(kNumRows); + + // write_batch_size=1 + a byte-based data page size makes the int32 and int64 leaves + // flush pages at different row counts, i.e. deliberately misaligned pages. + // max_row_group_length=kNumRows keeps all rows in a single row group. + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/kNumRows, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + int64_t total = 0; + auto top = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + for (int64_t i = 0; i < key_arr->length(); ++i) { + int64_t k = key_arr->Value(i); + // Full-row deep compare across ALL columns (struct + list + map): the returned + // row must equal the original row identified by key k. This is what catches a + // desync in the repeated list/map leaves, whose reassembly also relies on the + // per-leaf skip/read staying row-consistent. + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + ++total; + } + + for (int64_t i = 0; i < 15; ++i) { + ASSERT_EQ(i, key_arr->Value(i)); + } + for (int64_t i = 0; i < 10; ++i) { + ASSERT_EQ(77 + i, key_arr->Value(15 + i)); + } + + ASSERT_EQ(total, 25); +} + +/// Test: same misaligned nested layout as the single-row-group case above, but split +/// across MULTIPLE row groups (max_row_group_length=40 -> row groups of 40/40/20). +/// The selection bitmap spans row-group boundaries so the reader must keep the +/// per-leaf skip/read row-consistent both across misaligned pages and across row +/// groups. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows -> keys 0..14 and 77..86 +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesMultiRowGroup) { + std::string file_name = dir_->Str() + "/nested_misaligned_multi_rg.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedData(kNumRows); + + // write_batch_size=1 + byte-based data page size -> misaligned leaf pages. + // max_row_group_length=40 -> 3 row groups (40, 40, 20). + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) -> spans all three row groups. + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + std::vector keys; + + auto concated = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto top = std::dynamic_pointer_cast(concated); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + ASSERT_TRUE(key_arr); + for (int64_t i = 0; i < key_arr->length(); ++i) { + int64_t k = key_arr->Value(i); + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + keys.push_back(k); + } + + std::vector expected; + for (int64_t i = 0; i < 15; ++i) { + expected.push_back(i); + } + for (int64_t i = 77; i < 87; ++i) { + expected.push_back(i); + } + ASSERT_EQ(keys, expected); +} + +/// Helper: like MakeMisalignedNestedData but sprinkles nulls into the nested columns +/// so that definition levels vary per row (which also perturbs page boundaries): +/// key: int64, always non-null (= i, used to identify the row) +/// s: struct; whole struct null when i%11==0, otherwise +/// x null when i%5==0 and y null when i%7==0 +/// tags: list; null when i%6==0, otherwise [i*10, i*10+1] +/// props: map; null when i%8==0, otherwise {"k_i": i*100} +/// Correctness is verified per row by deep-comparing against this array. +static std::shared_ptr MakeMisalignedNestedDataWithNulls(int32_t num_rows) { + arrow::Int64Builder key_builder; + EXPECT_TRUE(key_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + key_builder.UnsafeAppend(i); + } + auto key_array = key_builder.Finish().ValueOrDie(); + + // s: struct with nulls at both leaf and struct level. + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int64()); + auto x_builder = std::make_shared(); + auto y_builder = std::make_shared(); + arrow::StructBuilder s_builder(arrow::struct_({field_x, field_y}), arrow::default_memory_pool(), + {x_builder, y_builder}); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 11 == 0) { + // AppendNull() also appends nulls to the child builders, keeping lengths in sync. + EXPECT_TRUE(s_builder.AppendNull().ok()); + continue; + } + EXPECT_TRUE(s_builder.Append().ok()); + if (i % 5 == 0) { + EXPECT_TRUE(x_builder->AppendNull().ok()); + } else { + EXPECT_TRUE(x_builder->Append(i).ok()); + } + if (i % 7 == 0) { + EXPECT_TRUE(y_builder->AppendNull().ok()); + } else { + EXPECT_TRUE(y_builder->Append(i).ok()); + } + } + auto s_array = s_builder.Finish().ValueOrDie(); + + // tags: list with null lists. + auto item_builder = std::make_shared(); + arrow::ListBuilder tags_builder(arrow::default_memory_pool(), item_builder); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 6 == 0) { + EXPECT_TRUE(tags_builder.AppendNull().ok()); + } else { + EXPECT_TRUE(tags_builder.Append().ok()); + EXPECT_TRUE(item_builder->Append(i * 10).ok()); + EXPECT_TRUE(item_builder->Append(i * 10 + 1).ok()); + } + } + auto tags_array = tags_builder.Finish().ValueOrDie(); + + // props: map with null maps. + auto map_key_builder = std::make_shared(); + auto map_val_builder = std::make_shared(); + arrow::MapBuilder props_builder(arrow::default_memory_pool(), map_key_builder, map_val_builder); + for (int32_t i = 0; i < num_rows; ++i) { + if (i % 8 == 0) { + EXPECT_TRUE(props_builder.AppendNull().ok()); + } else { + EXPECT_TRUE(props_builder.Append().ok()); + EXPECT_TRUE(map_key_builder->Append("k_" + std::to_string(i)).ok()); + EXPECT_TRUE(map_val_builder->Append(i * 100).ok()); + } + } + auto props_array = props_builder.Finish().ValueOrDie(); + + auto field_key = arrow::field("key", arrow::int64()); + auto field_s = arrow::field("s", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + return arrow::StructArray::Make({key_array, s_array, tags_array, props_array}, + {field_key, field_s, field_tags, field_props}) + .ValueOrDie(); +} + +/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split across +/// MULTIPLE row groups. This combines the three stress dimensions: null-driven +/// definition levels, byte-based misaligned pages, and row-group boundaries that the +/// selection bitmap crosses. Every returned row is deep-compared (nulls included) +/// against the original data identified by its non-null key. +/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling) +/// Expected: 25 rows -> keys 0..14 and 77..86 +TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMultiRowGroup) { + std::string file_name = dir_->Str() + "/nested_nulls_misaligned_multi_rg.parquet"; + constexpr int32_t kNumRows = 100; + auto data = MakeMisalignedNestedDataWithNulls(kNumRows); + + // write_batch_size=1 + byte-based data page size -> misaligned leaf pages. + // max_row_group_length=40 -> 3 row groups (40, 40, 20). + WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40, + /*enable_dictionary=*/false, /*data_page_size=*/40); + + auto read_schema = + arrow::schema({arrow::field("key", arrow::int64()), + arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()), + arrow::field("y", arrow::int64())})), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + // bitmap: [0,15), [77, 87) -> spans all three row groups. + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 15); + bitmap.AddRange(77, 87); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + ASSERT_TRUE(result); + + std::vector keys; + + auto concated = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto top = std::dynamic_pointer_cast(concated); + ASSERT_TRUE(top); + auto key_arr = std::dynamic_pointer_cast(top->field(0)); + ASSERT_TRUE(key_arr); + for (int64_t i = 0; i < key_arr->length(); ++i) { + ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null"; + int64_t k = key_arr->Value(i); + // Deep compare including nulls across struct/list/map leaves. + ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1))) + << "row content mismatch at key " << k; + keys.push_back(k); + } + + std::vector expected; + for (int64_t i = 0; i < 15; ++i) { + expected.push_back(i); + } + for (int64_t i = 77; i < 87; ++i) { + expected.push_back(i); + } + ASSERT_EQ(keys, expected); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index d7a1cde1..4f1d013f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include #include #include @@ -40,8 +41,13 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_schema_util.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" @@ -61,110 +67,175 @@ namespace paimon::parquet { ParquetFileBatchReader::ParquetFileBatchReader( std::shared_ptr&& input_stream, std::unique_ptr&& reader, const std::map& options, - const std::shared_ptr& arrow_pool) + const std::shared_ptr& arrow_pool, + std::shared_ptr> storage_read_bytes) : options_(options), arrow_pool_(arrow_pool), input_stream_(std::move(input_stream)), reader_(std::move(reader)), - read_ranges_(reader_->GetAllRowGroupRanges()), - metrics_(std::make_shared()) {} + metrics_(std::make_shared()), + storage_read_bytes_(std::move(storage_read_bytes)), + logger_(Logger::GetLogger("ParquetFileBatchReader")) {} Result> ParquetFileBatchReader::Create( std::shared_ptr&& input_stream, - const std::shared_ptr& pool, - const std::map& options, int32_t batch_size) { - assert(input_stream); - PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties, - CreateReaderProperties(pool, options)); - PAIMON_ASSIGN_OR_RAISE(::parquet::ArrowReaderProperties arrow_reader_properties, - CreateArrowReaderProperties(pool, options, batch_size)); - - ::parquet::arrow::FileReaderBuilder file_reader_builder; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.Open(input_stream, reader_properties)); - - std::unique_ptr<::parquet::arrow::FileReader> file_reader; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(pool.get()) - ->properties(arrow_reader_properties) - ->Build(&file_reader)); - - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - FileReaderWrapper::Create(std::move(file_reader))); - auto parquet_file_batch_reader = std::unique_ptr( - new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, - parquet_file_batch_reader->GetFileSchema()); - PAIMON_RETURN_NOT_OK(parquet_file_batch_reader->SetReadSchema( - file_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); - return parquet_file_batch_reader; + const std::map& options, int32_t batch_size, + std::shared_ptr<::parquet::FileMetaData> file_metadata, + std::shared_ptr> storage_read_bytes, + const std::shared_ptr& pool) { + try { + assert(input_stream); + PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties, + CreateReaderProperties(pool, options)); + + PAIMON_ASSIGN_OR_RAISE(::parquet::ArrowReaderProperties arrow_reader_properties, + CreateArrowReaderProperties(pool, options, batch_size)); + + ::parquet::arrow::FileReaderBuilder file_reader_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + file_reader_builder.Open(input_stream, reader_properties, std::move(file_metadata))); + + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(pool.get()) + ->properties(arrow_reader_properties) + ->Build(&file_reader)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + FileReaderWrapper::Create(std::move(file_reader), + static_cast(batch_size), pool)); + auto parquet_file_batch_reader = std::unique_ptr( + new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool, + std::move(storage_read_bytes))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, + parquet_file_batch_reader->GetFileSchema()); + PAIMON_RETURN_NOT_OK(parquet_file_batch_reader->SetReadSchema( + file_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); + return parquet_file_batch_reader; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::Create") } Result> ParquetFileBatchReader::GetFileSchema() const { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, - ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr new_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); + try { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, + ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr new_type, + ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); - auto c_schema = std::make_unique<::ArrowSchema>(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); - return c_schema; + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); + return c_schema; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GetFileSchema") } Status ParquetFileBatchReader::SetReadSchema( ::ArrowSchema* schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - if (!schema) { - return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr"); - } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, - arrow::ImportSchema(schema)); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - std::unordered_map> field_index_map; - int32_t i = 0; - for (const auto& field : file_schema->fields()) { - std::vector v; - FlattenSchema(field->type(), &i, &v); - field_index_map[field->name()] = v; - } - - std::vector column_indices; - for (const auto& field : read_schema->field_names()) { - if (field_index_map.find(field) != field_index_map.end()) { - for (int32_t index : field_index_map[field]) { - column_indices.push_back(index); - } - } else { - return Status::Invalid(fmt::format("Field {} is not found in schema.", field)); + try { + if (!schema) { + return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr"); } - } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, + arrow::ImportSchema(schema)); - std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); - if (predicate) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByPredicate(predicate, file_schema, row_groups)); - } - if (selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByBitmap(selection_bitmap.value(), row_groups)); - } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - read_data_type_ = arrow::struct_(read_schema->fields()); - read_row_groups_ = row_groups; - read_column_indices_ = column_indices; + // Recursively match read_schema against file_schema by field names. + // STRUCT supports sub-field projection; LIST/MAP require exact type match. + PAIMON_ASSIGN_OR_RAISE(std::vector column_indices, + ComputeNestedColumnIndices(read_schema, file_schema)); + + // Build column name to index map for page-level filtering. + // We still need the full per-top-level-field leaf indices for predicate pushdown. + std::unordered_map> field_index_map; + int32_t flat_idx = 0; + for (const auto& field : file_schema->fields()) { + std::vector leaf_indices; + FlattenSchema(field->type(), &flat_idx, &leaf_indices); + field_index_map[field->name()] = leaf_indices; + } - metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, reader_->GetNumberOfRowGroups()); - metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size()); + TargetRowGroups target_row_groups = + TargetRowGroup::MakeForAllRowGroups(reader_->GetAllRowGroupRanges()); + PAIMON_ASSIGN_OR_RAISE( + bool enable_page_index_filter, + OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, + DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); - PAIMON_ASSIGN_OR_RAISE(std::set ordered_row_groups, - reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); - return reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + if (predicate) { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + FilterRowGroupsByPredicate(predicate, file_schema, target_row_groups)); + } + if (selection_bitmap) { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + FilterRowGroupsByBitmap(selection_bitmap.value(), target_row_groups)); + if (enable_page_index_filter) { + // To decide which strategy to use, "trim" or "coalesce". "Coalesce" By default. + PAIMON_ASSIGN_OR_RAISE( + std::string strategy, + OptionsUtils::GetValueFromMap( + options_, PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY, + DEFAULT_PARQUET_READ_BITMAP_STRATEGY)); + if (strategy == "trim") { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + RefineRowRangesByTrimming(selection_bitmap.value(), target_row_groups, + column_indices)); + } else if (strategy == "coalesce") { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + RefineRowRangesByCoalescing(selection_bitmap.value(), target_row_groups)); + } else { + return Status::Invalid( + fmt::format("Invalid row range refining strategy :{}, valid strategies " + "are: trim, coalesce", + strategy)); + } + } + } + // Apply page-level filtering after bitmap pruning so we don't read page index + // pages for row groups that the bitmap already excluded. + // If no predicate is provided, skip page-level filtering + if (predicate && !target_row_groups.empty()) { + if (enable_page_index_filter) { + // Build column name to index map for page-level filtering. + // For leaf columns, indices[0] is the correct leaf column index in Parquet. + // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, + // but predicate pushdown only targets leaf columns with simple types, so indices[0] + // is always the correct single leaf index for predicate evaluation. + std::map column_name_to_index; + for (const auto& [name, indices] : field_index_map) { + if (!indices.empty()) { + column_name_to_index[name] = indices[0]; + } + } + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + FilterRowGroupsByPageIndex(predicate, column_name_to_index, target_row_groups)); + } + } + + read_data_type_ = arrow::struct_(read_schema->fields()); + + metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, + reader_->GetNumberOfRowGroups()); + metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, + target_row_groups.size()); + + PAIMON_RETURN_NOT_OK(UpdateAllTargetRowRanges(target_row_groups)); + PAIMON_RETURN_NOT_OK(reader_->PrepareForReadingLazy(target_row_groups, column_indices)); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::SetReadSchema") + return Status::OK(); } -Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( +Result ParquetFileBatchReader::FilterRowGroupsByPredicate( const std::shared_ptr& predicate, const std::shared_ptr file_schema, - const std::vector& src_row_groups) const { + const TargetRowGroups& src_row_groups) const { if (!predicate) { return Status::Invalid("cannot pushdown an empty predicate"); } @@ -187,81 +258,321 @@ Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( std::shared_ptr file_fragment, parquet_file_format->MakeFragment( file_source, /*partition_expression=*/PredicateConverter::AlwaysTrue(), - /*physical_schema=*/nullptr, /*row_groups=*/src_row_groups)); + /*physical_schema=*/nullptr, + /*row_groups=*/TargetRowGroup::GetRowGroupIndices(src_row_groups))); PAIMON_RETURN_NOT_OK_FROM_ARROW( file_fragment->EnsureCompleteMetadata(reader_->GetFileReader())); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow::dataset::FragmentVector target_fragments, file_fragment->SplitByRowGroup(bind_expr)); - std::vector target_row_groups; + TargetRowGroups target_row_groups; target_row_groups.reserve(src_row_groups.size()); for (const auto& fragment : target_fragments) { auto parquet_fragment = dynamic_cast(fragment.get()); if (!parquet_fragment) { return Status::Invalid("cannot cast to ParquetFileFragment in ParquetFileBatchReader"); } - target_row_groups.insert(target_row_groups.end(), parquet_fragment->row_groups().begin(), - parquet_fragment->row_groups().end()); + for (auto rg_index : parquet_fragment->row_groups()) { + for (const auto& row_group : src_row_groups) { + if (row_group.GetRowGroupIndex() == rg_index) { + target_row_groups.emplace_back(row_group); + break; + } + } + } } return target_row_groups; } -Result> ParquetFileBatchReader::FilterRowGroupsByBitmap( - const RoaringBitmap32& bitmap, const std::vector& src_row_groups) const { +Result ParquetFileBatchReader::FilterRowGroupsByBitmap( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const { if (bitmap.IsEmpty()) { return Status::Invalid("cannot push down an empty bitmap to ParquetFileBatchReader"); } + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); - // filter row groups by row range - std::vector target_row_groups; - for (const auto& row_group_idx : src_row_groups) { + + TargetRowGroups target_row_groups; + for (const auto& row_group : src_row_groups) { + int32_t row_group_idx = row_group.GetRowGroupIndex(); if (static_cast(row_group_idx) >= all_row_group_ranges.size()) { return Status::Invalid( fmt::format("src row group {} not in row group meta", row_group_idx)); } + // half open interval [start_row_idx, end_row_idx) const auto& [start_row_idx, end_row_idx] = all_row_group_ranges[row_group_idx]; - if (bitmap.ContainsAny(start_row_idx, end_row_idx)) { - target_row_groups.push_back(row_group_idx); + if (!bitmap.ContainsAny(start_row_idx, end_row_idx)) { + continue; } + target_row_groups.emplace_back(row_group); } return target_row_groups; } -Result ParquetFileBatchReader::NextBatch() { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, reader_->Next()); - if (batch == nullptr) { - return BatchReader::MakeEofBatch(); - } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( - array, read_data_type_, arrow_pool_)); - } - PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - return Status::Invalid( - fmt::format("unexpected: in parquet, after CastArrayForTimestamp, output type {} not " - "equal with read schema {}", - array->type()->ToString(), read_data_type_->ToString())); +RowRanges ParquetFileBatchReader::CoalesceNearbyRanges(const RowRanges& input, + uint64_t hole_size_limit) { + if (input.IsEmpty()) { + return RowRanges(); + } + + const auto& ranges = input.GetRanges(); + RowRanges result; + int64_t merge_start = ranges.front().from; + int64_t merge_end = ranges.front().to; + + for (size_t i = 1; i < ranges.size(); ++i) { + // Gap between [merge_start, merge_end] and [ranges[i].from, ranges[i].to] + int64_t gap = ranges[i].from - merge_end - 1; + if (static_cast(gap) > hole_size_limit) { + result.Add(RowRanges::Range(merge_start, merge_end)); + merge_start = ranges[i].from; + } + merge_end = ranges[i].to; + } + result.Add(RowRanges::Range(merge_start, merge_end)); + return result; +} + +RowRanges ParquetFileBatchReader::BitmapToContiguousRanges(const RoaringBitmap32& bitmap, + uint64_t start_row, uint64_t end_row) { + RowRanges ranges; + if (bitmap.IsEmpty() || start_row >= end_row) { + return ranges; + } + + auto it = bitmap.EqualOrLarger(static_cast(start_row)); + const auto end = bitmap.End(); + if (it == end || static_cast(*it) >= end_row) { + return ranges; + } + + auto run_start = static_cast(*it); + auto prev = run_start; + + for (++it; it != end; ++it) { + auto current = static_cast(*it); + if (current >= static_cast(end_row)) { + break; + } + if (current != prev + 1) { + ranges.Add(RowRanges::Range(run_start - start_row, prev - start_row)); + run_start = current; + } + prev = current; + } + ranges.Add(RowRanges::Range(run_start - start_row, prev - start_row)); + return ranges; +} + +Result ParquetFileBatchReader::RefineRowRangesByCoalescing( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const { + PAIMON_ASSIGN_OR_RAISE(const uint64_t hole_size_limit, + OptionsUtils::GetValueFromMap( + options_, PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT, + DEFAULT_PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT)); + + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); + TargetRowGroups target_row_groups; + target_row_groups.reserve(src_row_groups.size()); + + for (const auto& row_group : src_row_groups) { + int32_t rg_index = row_group.GetRowGroupIndex(); + uint64_t rg_start_row = all_row_group_ranges[rg_index].first; + uint64_t rg_end_row = all_row_group_ranges[rg_index].second; + + // Step 1: bitmap -> contiguous ranges (relative to row group start). + // Step 2: coalesce ranges with small gaps to reduce range count. + RowRanges contiguous = BitmapToContiguousRanges(bitmap, rg_start_row, rg_end_row); + RowRanges coalesced = CoalesceNearbyRanges(contiguous, hole_size_limit); + + auto rg_row_count = static_cast(rg_end_row - rg_start_row); + if (coalesced.IsEmpty()) { + continue; + } + if (coalesced.RowCount() == rg_row_count) { + target_row_groups.emplace_back(row_group); + } else { + target_row_groups.emplace_back(rg_index, true, std::move(coalesced)); + } + } + return target_row_groups; +} + +Result ParquetFileBatchReader::RefineRowRangesByTrimming( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups, + const std::vector& column_indices) const { + auto page_index_reader = reader_->GetPageIndexReader(); + if (!page_index_reader) { + return src_row_groups; + } + + TargetRowGroups target_row_groups; + target_row_groups.reserve(src_row_groups.size()); + for (const auto& row_group : src_row_groups) { + auto filtered = + TrimRowGroupPageRanges(bitmap, row_group, column_indices, page_index_reader); + if (!filtered.GetRowRanges().IsEmpty()) { + target_row_groups.emplace_back(std::move(filtered)); + } + } + return target_row_groups; +} + +TargetRowGroup ParquetFileBatchReader::TrimRowGroupPageRanges( + const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, + const std::vector& column_indices, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const { + int32_t row_group_idx = row_group.GetRowGroupIndex(); + auto rg_page_index_reader = page_index_reader->RowGroup(row_group_idx); + if (!rg_page_index_reader) { + return row_group; + } + + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); + uint64_t rg_start_row = all_row_group_ranges[row_group_idx].first; + uint64_t rg_row_count = all_row_group_ranges[row_group_idx].second - rg_start_row; + + RowRanges row_ranges = row_group.GetRowRanges(); + for (int32_t col_index : column_indices) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_index); + if (!offset_index) { + continue; + } + auto page_ranges = ComputeColumnPageRanges(bitmap, offset_index->page_locations(), + rg_start_row, rg_row_count); + row_ranges = RowRanges::Intersection(row_ranges, page_ranges); + } + if (row_ranges.RowCount() == static_cast(rg_row_count)) { + return row_group; + } else { + return TargetRowGroup(row_group_idx, true, std::move(row_ranges)); } - std::unique_ptr c_array = std::make_unique(); - std::unique_ptr c_schema = std::make_unique(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); +} - read_rows_ += array->length(); - read_batch_count_++; - metrics_->SetCounter(ParquetMetrics::READ_ROWS, read_rows_); - metrics_->SetCounter(ParquetMetrics::READ_BATCH_COUNT, read_batch_count_); +RowRanges ParquetFileBatchReader::ComputeColumnPageRanges( + const RoaringBitmap32& bitmap, const std::vector<::parquet::PageLocation>& page_locations, + uint64_t rg_start_row, uint64_t rg_row_count) { + RowRanges page_row_ranges; + for (size_t page_idx = 0; page_idx < page_locations.size(); ++page_idx) { + // half open interval [first_row, last_row) + auto first_row = page_locations[page_idx].first_row_index; + auto last_row = page_idx + 1 < page_locations.size() + ? page_locations[page_idx + 1].first_row_index + : rg_row_count; - return make_pair(std::move(c_array), std::move(c_schema)); + if (!bitmap.ContainsAny(rg_start_row + first_row, rg_start_row + last_row)) { + continue; + } + // closed interval [range_start_row, range_end_row] + auto range_start_row = bitmap.NextValue(rg_start_row + first_row); + auto range_end_row = bitmap.PreviousValue(rg_start_row + last_row); + if (!range_start_row.has_value() || !range_end_row.has_value()) { + continue; + } + page_row_ranges.Add( + Range(range_start_row.value() - rg_start_row, range_end_row.value() - rg_start_row)); + } + return page_row_ranges; +} + +// Uses page-level column index statistics to filter row groups and store per-row-group +// RowRanges for true page-level skipping. A row group is excluded if ALL its pages are +// determined to not match the predicate. For partially matched row groups, RowRanges +// are stored for page-level filtering during reading. +Result ParquetFileBatchReader::FilterRowGroupsByPageIndex( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, + const TargetRowGroups& src_row_groups) const { + if (!predicate) { + return src_row_groups; + } + + auto page_index_reader = reader_->GetPageIndexReader(); + if (!page_index_reader) { + PAIMON_LOG_DEBUG(logger_, + "Page index not available in file, skipping page-level filtering (%s)", + PARQUET_WRITE_ENABLE_PAGE_INDEX); + return src_row_groups; + } + + auto file_metadata = reader_->GetFileReader()->parquet_reader()->metadata(); + + TargetRowGroups target_row_groups; + + for (const auto& row_group : src_row_groups) { + int32_t row_group_idx = row_group.GetRowGroupIndex(); + auto result = + reader_->CalculateFilteredRowRanges(row_group_idx, predicate, column_name_to_index); + + if (!result.ok()) { + target_row_groups.emplace_back(row_group); + continue; + } + + const auto& row_ranges = result.value(); + if (!row_ranges.IsEmpty()) { + int64_t rg_row_count = file_metadata->RowGroup(row_group_idx)->num_rows(); + auto intersection = row_group.IsPartiallyMatched() + ? RowRanges::Intersection(row_group.GetRowRanges(), row_ranges) + : row_ranges; + if (intersection.IsEmpty()) { + continue; + } + if (intersection.RowCount() < rg_row_count) { + target_row_groups.emplace_back(row_group_idx, true, intersection); + } else { + target_row_groups.emplace_back(row_group); + } + } + } + + return target_row_groups; +} + +Result ParquetFileBatchReader::NextBatch() { + try { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, reader_->Next()); + if (batch == nullptr) { + row_mapping_.clear(); + return BatchReader::MakeEofBatch(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + batch->ToStructArray()); + PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + array->type(), read_data_type_)); + if (need_cast) { + PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( + array, read_data_type_, arrow_pool_)); + } + PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + array->type(), read_data_type_)); + if (need_cast) { + return Status::Invalid(fmt::format( + "unexpected: in parquet, after CastArrayForTimestamp, output type {} not " + "equal with read schema {}", + array->type()->ToString(), read_data_type_->ToString())); + } + PAIMON_RETURN_NOT_OK(GenerateRowMapping(array->length())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); + + read_rows_ += array->length(); + read_batch_count_++; + metrics_->SetCounter(ParquetMetrics::READ_ROWS, read_rows_); + metrics_->SetCounter(ParquetMetrics::READ_BATCH_COUNT, read_batch_count_); + + return make_pair(std::move(c_array), std::move(c_schema)); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::NextBatch") } Result>> ParquetFileBatchReader::GenReadRanges( bool* need_prefetch) const { - *need_prefetch = true; - return reader_->GetAllRowGroupRanges(); + try { + *need_prefetch = true; + return reader_->GetAllRowGroupRanges(); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GenReadRanges") } Result<::parquet::ReaderProperties> ParquetFileBatchReader::CreateReaderProperties( @@ -307,16 +618,169 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead int64_t cache_prefetch_limit, OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT, DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT)); + PAIMON_ASSIGN_OR_RAISE( + int64_t cache_hole_size_limit, + OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, + DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT)); PAIMON_ASSIGN_OR_RAISE( int64_t cache_range_size_limit, OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT)); + if (cache_hole_size_limit < 0) { + return Status::Invalid(fmt::format("{} must be non-negative, but was {}", + PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, + cache_hole_size_limit)); + } + if (cache_range_size_limit <= cache_hole_size_limit) { + return Status::Invalid(fmt::format("{} must be greater than {}, but was {} <= {}", + PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, + PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, + cache_range_size_limit, cache_hole_size_limit)); + } auto cache_option = arrow::io::CacheOptions::Defaults(); cache_option.lazy = cache_lazy; cache_option.prefetch_limit = cache_prefetch_limit; + cache_option.hole_size_limit = cache_hole_size_limit; cache_option.range_size_limit = cache_range_size_limit; arrow_reader_props.set_cache_options(cache_option); return arrow_reader_props; } +// Nested column index computation + +Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& read_type, + const std::shared_ptr& file_type, + int32_t* leaf_index, + std::vector* indices) { + if (file_type->id() == arrow::Type::STRUCT) { + for (const auto& file_child : file_type->fields()) { + std::shared_ptr read_child = nullptr; + for (const auto& candidate : read_type->fields()) { + if (candidate->name() == file_child->name()) { + read_child = candidate; + break; + } + } + if (read_child) { + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_child->type(), file_child->type(), + leaf_index, indices)); + } else { + SkipLeafIndices(file_child->type(), leaf_index); + } + } + } else if (file_type->id() == arrow::Type::LIST || file_type->id() == arrow::Type::MAP) { + // Keep behavior aligned with ORC path: list/map inner partial projection + // is currently unsupported and should fail-fast. + if (!read_type->Equals(file_type)) { + return Status::Invalid(fmt::format( + "Parquet does not support partial projection inside list/map: src {} vs target {}", + file_type->ToString(), read_type->ToString())); + } + for (int32_t i = 0; i < file_type->num_fields(); i++) { + PAIMON_RETURN_NOT_OK(CollectLeafIndices( + read_type->field(i)->type(), file_type->field(i)->type(), leaf_index, indices)); + } + } else { + // Leaf column — collect its index. + indices->push_back((*leaf_index)++); + } + return Status::OK(); +} + +void ParquetFileBatchReader::SkipLeafIndices(const std::shared_ptr& file_type, + int32_t* leaf_index) { + if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST || + file_type->id() == arrow::Type::MAP) { + for (int32_t i = 0; i < file_type->num_fields(); i++) { + SkipLeafIndices(file_type->field(i)->type(), leaf_index); + } + } else { + (*leaf_index)++; + } +} + +Result> ParquetFileBatchReader::ComputeNestedColumnIndices( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema) { + std::vector indices; + std::vector file_field_leaf_starts; + file_field_leaf_starts.reserve(file_schema->num_fields()); + + int32_t file_leaf_index = 0; + for (const auto& file_field : file_schema->fields()) { + file_field_leaf_starts.push_back(file_leaf_index); + SkipLeafIndices(file_field->type(), &file_leaf_index); + } + + const auto& file_fields = file_schema->fields(); + for (const auto& read_field : read_schema->fields()) { + int32_t file_field_idx = -1; + for (int32_t i = 0; i < static_cast(file_fields.size()); ++i) { + if (file_fields[i]->name() == read_field->name()) { + file_field_idx = i; + break; + } + } + if (file_field_idx < 0) { + return Status::Invalid( + fmt::format("Field '{}' in read schema does not exist in parquet file schema", + read_field->name())); + } + int32_t leaf_index = file_field_leaf_starts[file_field_idx]; + PAIMON_RETURN_NOT_OK(CollectLeafIndices( + read_field->type(), file_fields[file_field_idx]->type(), &leaf_index, &indices)); + } + return indices; +} + +Status ParquetFileBatchReader::UpdateAllTargetRowRanges( + const std::vector& target_row_groups) { + row_mapping_.clear(); + auto all_row_group_ranges = reader_->GetAllRowGroupRanges(); + RowRanges all_ranges; + for (const auto& target_row_group : target_row_groups) { + auto row_group_idx = target_row_group.GetRowGroupIndex(); + for (const auto& range : target_row_group.GetRowRanges().GetRanges()) { + all_ranges.Add(Range(all_row_group_ranges[row_group_idx].first + range.from, + all_row_group_ranges[row_group_idx].first + range.to)); + } + } + all_row_ranges_ = std::move(all_ranges); + return Status::OK(); +} + +Status ParquetFileBatchReader::GenerateRowMapping(int64_t batch_length) { + const std::vector& all_ranges = all_row_ranges_.GetRanges(); + PAIMON_ASSIGN_OR_RAISE(int64_t batch_start_row, reader_->GetPreviousBatchFirstRowNumber()); + + auto cur_range_it = + std::upper_bound(all_ranges.begin(), all_ranges.end(), batch_start_row, + [](int64_t value, const Range& r) { return value < r.from; }); + if (cur_range_it == all_ranges.begin()) { + return Status::Invalid("No range found!"); + } + --cur_range_it; + if (batch_start_row < cur_range_it->from || batch_start_row > cur_range_it->to) { + return Status::Invalid( + fmt::format("Batch start row {} is not in the current range [{}, {}]!", batch_start_row, + cur_range_it->from, cur_range_it->to)); + } + + std::vector row_mapping; + row_mapping.reserve(batch_length); + int64_t global_row = batch_start_row; + for (int64_t i = 0; i < batch_length; ++i) { + if (global_row > cur_range_it->to) { + ++cur_range_it; + if (cur_range_it == all_ranges.end()) { + return Status::Invalid("Batch length exceeds the total row ranges!"); + } + global_row = cur_range_it->from; + } + row_mapping.push_back(global_row); + global_row++; + } + row_mapping_ = std::move(row_mapping); + return Status::OK(); +} } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 806afbe7..0cdecbfb 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -18,6 +18,9 @@ #pragma once +#include + +#include #include #include #include @@ -38,6 +41,10 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/file_reader_wrapper.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" +#include "paimon/logging.h" #include "paimon/reader/prefetch_file_batch_reader.h" #include "paimon/result.h" #include "paimon/status.h" @@ -51,6 +58,9 @@ namespace io { class RandomAccessFile; } // namespace io } // namespace arrow +namespace parquet { +class FileMetaData; +} // namespace parquet namespace paimon { class Metrics; class Predicate; @@ -63,8 +73,14 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { public: static Result> Create( std::shared_ptr&& input_stream, + const std::map& options, int32_t batch_size, + std::shared_ptr<::parquet::FileMetaData> file_metadata, + std::shared_ptr> storage_read_bytes, + const std::shared_ptr& pool); + + static Result<::parquet::ReaderProperties> CreateReaderProperties( const std::shared_ptr& pool, - const std::map& options, int32_t batch_size); + const std::map& options); // For timestamp type, we return the schema stored in file, e.g., second in parquet file will // store as milli. @@ -86,9 +102,22 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { Result>> GenReadRanges( bool* need_prefetch) const override; - Result GetPreviousBatchFirstRowNumber() const override { - assert(reader_); - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (row_mapping_.empty()) { + PAIMON_ASSIGN_OR_RAISE(uint64_t previous_first_row, + reader_->GetPreviousBatchFirstRowNumber()); + if (previous_first_row == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= row_mapping_.size()) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, row_mapping_.size())); + } + return row_mapping_[batch_row_id]; } Result GetNumberOfRows() const override { @@ -102,14 +131,12 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } Status SetReadRanges(const std::vector>& read_ranges) override { - read_ranges_ = read_ranges; - PAIMON_ASSIGN_OR_RAISE( - std::set ordered_row_groups, - reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); - return reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + return reader_->ApplyReadRanges(read_ranges); } std::shared_ptr GetReaderMetrics() const override { + uint64_t storage = storage_read_bytes_ ? storage_read_bytes_->load() : 0; + metrics_->SetCounter(ParquetMetrics::READ_STORAGE_BYTES, storage); return metrics_; } @@ -130,11 +157,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { ParquetFileBatchReader(std::shared_ptr&& input_stream, std::unique_ptr&& reader, const std::map& options, - const std::shared_ptr& arrow_pool); - - static Result<::parquet::ReaderProperties> CreateReaderProperties( - const std::shared_ptr& pool, - const std::map& options); + const std::shared_ptr& arrow_pool, + std::shared_ptr> storage_read_bytes); static Result<::parquet::ArrowReaderProperties> CreateArrowReaderProperties( const std::shared_ptr& pool, @@ -154,14 +178,75 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } } + /// Recursively collect leaf column indices for the sub-fields in read_type + /// that match file_type by paimon field ID. Unmatched sub-fields in file_type + /// have their leaf indices skipped. Partial projection inside LIST/MAP is + /// not supported and will return Invalid. + static Status CollectLeafIndices(const std::shared_ptr& read_type, + const std::shared_ptr& file_type, + int32_t* leaf_index, std::vector* indices); + + /// Skip over all leaf column indices of the given file_type without collecting. + static void SkipLeafIndices(const std::shared_ptr& file_type, + int32_t* leaf_index); + + /// Compute leaf column indices by recursively matching read_schema against + /// file_schema using paimon field IDs. STRUCT supports sub-field projection + /// (unmatched sub-fields are skipped). LIST/MAP require exact type match. + static Result> ComputeNestedColumnIndices( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema); + + Status UpdateAllTargetRowRanges(const std::vector& target_row_groups); + // precondition: predicate supposed not be empty - Result> FilterRowGroupsByPredicate( + Result FilterRowGroupsByPredicate( const std::shared_ptr& predicate, const std::shared_ptr file_schema, - const std::vector& src_row_groups) const; + const TargetRowGroups& src_row_groups) const; + + Result FilterRowGroupsByBitmap(const RoaringBitmap32& bitmap, + const TargetRowGroups& src_row_groups) const; + + // Apply bitmap filtering to row ranges by trimming start and end rows in pages. + // Then apply intersection among all target columns. + Result RefineRowRangesByTrimming( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups, + const std::vector& column_indices) const; + + // Apply page-level bitmap filtering to a single row group across all + // requested columns. Intersects the row group's existing ranges with the + // per-column page ranges derived from the bitmap. + TargetRowGroup TrimRowGroupPageRanges( + const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, + const std::vector& column_indices, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const; + + // Apply bitmap filtering to row ranges by coalescing nearby ranges. + Result RefineRowRangesByCoalescing( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const; + // Convert bitmap set bits within [start_row, end_row) to contiguous + // row ranges, stored relative to start_row. + static RowRanges BitmapToContiguousRanges(const RoaringBitmap32& bitmap, uint64_t start_row, + uint64_t end_row); + // Merge ranges whose inter-range gap is <= hole_size_limit. + static RowRanges CoalesceNearbyRanges(const RowRanges& input, uint64_t hole_size_limit); + + // Compute the set of row ranges within a single column's pages that + // overlap with the given bitmap. For each page, the bitmap is queried to + // find the first/last matching row in each page, used to trim the page head/tail + static RowRanges ComputeColumnPageRanges( + const RoaringBitmap32& bitmap, const std::vector<::parquet::PageLocation>& page_locations, + uint64_t rg_start_row, uint64_t rg_row_count); + + // Apply page-level filtering using column index. + // Returns (filtered row groups, per-row-group RowRanges for partial matches). + Result FilterRowGroupsByPageIndex( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, + const TargetRowGroups& src_row_groups) const; - Result> FilterRowGroupsByBitmap( - const RoaringBitmap32& bitmap, const std::vector& src_row_groups) const; + Status GenerateRowMapping(int64_t batch_length); private: std::map options_; @@ -172,16 +257,17 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { std::unique_ptr reader_; std::shared_ptr read_data_type_; - std::vector> read_ranges_; std::shared_ptr metrics_; + // storageReadBytes counter shared with the underlying ArrowInputStreamAdapter. + std::shared_ptr> storage_read_bytes_; + std::unique_ptr logger_; uint64_t read_rows_ = 0; uint64_t read_batch_count_ = 0; - // last time set read schema - std::vector read_row_groups_; - std::vector read_column_indices_; + RowRanges all_row_ranges_; + std::vector row_mapping_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 836c291b..dd323460 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -18,8 +18,12 @@ #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include +#include #include #include +#include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -30,22 +34,27 @@ #include "arrow/c/bridge.h" #include "arrow/io/caching.h" #include "arrow/io/interfaces.h" +#include "arrow/ipc/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/defs.h" +#include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/format/parquet/parquet_reader_builder.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/reader/batch_reader.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/testing/utils/timezone_guard.h" @@ -58,9 +67,69 @@ class Predicate; namespace paimon::parquet::test { +std::string SerializeSchemaToString(const std::shared_ptr& schema) { + std::shared_ptr serialized = arrow::ipc::SerializeSchema(*schema).ValueOrDie(); + return std::string(reinterpret_cast(serialized->data()), + static_cast(serialized->size())); +} + +class FailedUriInputStream : public InputStream { + public: + explicit FailedUriInputStream(const std::shared_ptr& input) : input_(input) {} + + Status Seek(int64_t offset, SeekOrigin origin) override { + return input_->Seek(offset, origin); + } + + Result GetPos() const override { + return input_->GetPos(); + } + + Result Read(char* buffer, int64_t size) override { + return input_->Read(buffer, size); + } + + Result Read(char* buffer, int64_t size, int64_t offset) override { + return input_->Read(buffer, size, offset); + } + + void ReadAsync(char* buffer, int64_t size, int64_t offset, + std::function&& callback) override { + return input_->ReadAsync(buffer, size, offset, std::move(callback)); + } + + Result GetUri() const override { + return Status::Invalid("failed to get uri"); + } + + Result Length() const override { + return input_->Length(); + } + + Status Close() override { + return input_->Close(); + } + + private: + std::shared_ptr input_; +}; + class ParquetFileBatchReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: + static std::shared_ptr WithMapSelectedKeys( + const std::shared_ptr& field, const std::string& selected_keys) { + auto metadata = + field->metadata() ? field->metadata()->Copy() : arrow::key_value_metadata({}); + auto set_status = metadata->Set(DataField::MAP_SELECTED_KEYS, selected_keys); + EXPECT_TRUE(set_status.ok()) << set_status.ToString(); + return field->WithMetadata(metadata); + } + + static std::shared_ptr MakeReadSchema(const arrow::FieldVector& fields) { + return arrow::schema(fields); + } + void SetUp() override { dir_ = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); @@ -103,12 +172,15 @@ class ParquetFileBatchReaderTest : public ::testing::Test, void WriteArray(const std::string& file_path, const std::shared_ptr& src_array, const std::shared_ptr& arrow_schema, int64_t write_batch_size, - bool enable_dictionary, int64_t max_row_group_length) const { + bool enable_dictionary, int64_t max_row_group_length, + int64_t max_page_size = 1024 * 1024 * 1024) const { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/true)); ::parquet::WriterProperties::Builder builder; builder.write_batch_size(write_batch_size); builder.max_row_group_length(max_row_group_length); + builder.data_pagesize(max_page_size); + builder.enable_write_page_index(); enable_dictionary ? builder.enable_dictionary() : builder.disable_dictionary(); auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN(auto format_writer, ParquetFormatWriter::Create( @@ -127,14 +199,19 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::unique_ptr PrepareParquetFileBatchReader( const std::string& file_name, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, - const std::optional& selection_bitmap, int32_t batch_size) const { + const std::optional& selection_bitmap, int32_t batch_size, + bool enable_page_level_filter = false) const { EXPECT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_name)); auto length = fs_->GetFileStatus(file_name).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), pool_, length); - std::map options = {}; + std::make_unique(std::move(input_stream), length, pool_); + auto storage_read_bytes = in_stream->StorageReadBytes(); + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = + enable_page_level_filter ? "true" : "false"; return PrepareParquetFileBatchReader(std::move(in_stream), options, read_schema, predicate, - selection_bitmap, batch_size); + selection_bitmap, batch_size, + std::move(storage_read_bytes)); } std::unique_ptr PrepareParquetFileBatchReader( @@ -142,10 +219,12 @@ class ParquetFileBatchReaderTest : public ::testing::Test, const std::map& options, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, - const std::optional& selection_bitmap, int32_t batch_size) const { - EXPECT_OK_AND_ASSIGN( - auto parquet_batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)); + const std::optional& selection_bitmap, int32_t batch_size, + std::shared_ptr> storage_read_bytes = nullptr) const { + EXPECT_OK_AND_ASSIGN(auto parquet_batch_reader, + ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, std::move(storage_read_bytes), pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); EXPECT_TRUE(arrow_status.ok()); @@ -153,7 +232,7 @@ class ParquetFileBatchReaderTest : public ::testing::Test, return parquet_batch_reader; } - private: + protected: std::string file_path_; std::unique_ptr dir_; std::shared_ptr fs_; @@ -163,6 +242,128 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::shared_ptr struct_array_; }; +static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder val_builder; + EXPECT_TRUE(val_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + val_builder.UnsafeAppend(i); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("f0", arrow::int32()); + return arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); +} + +TEST_F(ParquetFileBatchReaderTest, TestParquetMetadataCacheReusesSerializedFooter) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/struct_array_->length(), + /*enable_dictionary=*/false, + /*max_row_group_length=*/struct_array_->length()); + + auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, + 128 * 1024 * 1024); + auto open_reader = [&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, fs_->Open(file_path_)); + std::map options; + ParquetReaderBuilder builder(options, batch_size_); + builder.WithMemoryPool(GetDefaultPool())->WithCache(cache); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + builder.Build(input_stream)); + auto parquet_reader = dynamic_cast(reader.release()); + if (parquet_reader == nullptr) { + return Status::Invalid("failed to cast FileBatchReader to ParquetFileBatchReader"); + } + return std::unique_ptr(parquet_reader); + }; + + ASSERT_OK_AND_ASSIGN(auto reader1, open_reader()); + ASSERT_OK_AND_ASSIGN(auto schema1, reader1->GetFileSchema()); + ASSERT_TRUE(schema1); + ASSERT_TRUE(schema1->release); + schema1->release(schema1.get()); + ASSERT_EQ(1, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); + + ASSERT_OK_AND_ASSIGN(auto reader2, open_reader()); + ASSERT_OK_AND_ASSIGN(auto schema2, reader2->GetFileSchema()); + ASSERT_TRUE(schema2); + ASSERT_TRUE(schema2->release); + schema2->release(schema2.get()); + ASSERT_EQ(2, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); +} + +TEST_F(ParquetFileBatchReaderTest, TestParquetMetadataCacheBypassesWhenGetUriFails) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/struct_array_->length(), + /*enable_dictionary=*/false, + /*max_row_group_length=*/struct_array_->length()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_path_)); + auto failed_uri_input_stream = std::make_shared(input_stream); + auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, + 128 * 1024 * 1024); + + std::map options; + ParquetReaderBuilder builder(options, batch_size_); + builder.WithMemoryPool(GetDefaultPool())->WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + builder.Build(failed_uri_input_stream)); + auto parquet_reader = dynamic_cast(reader.get()); + ASSERT_TRUE(parquet_reader); + ASSERT_OK_AND_ASSIGN(auto file_schema, parquet_reader->GetFileSchema()); + ASSERT_TRUE(file_schema); + ASSERT_TRUE(file_schema->release); + file_schema->release(file_schema.get()); + + ASSERT_EQ(0, cache->GetCount()); + ASSERT_EQ(0, cache->SupplierCallCount()); + ASSERT_EQ(0, cache->Size()); +} + +TEST_F(ParquetFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_schema = arrow::schema({write_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({write_field}), data_json) + .ValueOrDie()); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), file_name); + WriteArray(file_path, write_array, write_schema, /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto read_field = arrow::field("f0", arrow::binary()); + auto read_schema = arrow::schema({read_field}); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + ASSERT_TRUE(file_schema->Equals(*read_schema)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_field}), data_json) + .ValueOrDie()); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.parquet"); + check_binary_read_result(arrow::large_binary(), "large-binary.parquet"); +} + TEST_F(ParquetFileBatchReaderTest, TestSimple) { std::string file_name = paimon::test::GetDataDir() + "/parquet/parquet_append_table.db/parquet_append_table/bucket-0/" @@ -185,11 +386,12 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_name)); auto length = fs_->GetFileStatus(file_name).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), pool_, length); + std::make_unique(std::move(input_stream), length, pool_); std::map options; - ASSERT_OK_AND_ASSIGN( - auto parquet_batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size_)); + ASSERT_OK_AND_ASSIGN(auto parquet_batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size_, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, pool_)); // test GetFileSchema() ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); auto arrow_file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); @@ -232,6 +434,36 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) { ASSERT_FALSE(result_with_read_schema); } +TEST_F(ParquetFileBatchReaderTest, TestSetReadSchemaWithLegacyParquetMissingFieldIds) { + std::string file_name = paimon::test::GetDataDir() + + "/parquet/append_09.db/append_09/f1=20/bucket-0/" + "data-b446f78a-2cfb-4b3b-add8-31295d24a277-0.parquet"; + + std::vector read_fields = { + DataField(0, arrow::field("f0", arrow::utf8())), + DataField(2, arrow::field("f2", arrow::int32())), + DataField(3, arrow::field("f3", arrow::float64())), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_name, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(read_schema->fields()), {R"([ + ["Lucy", 1, 14.1] + ])"}, + &expected_array) + .ok()); + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { std::string file_name = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" @@ -240,20 +472,17 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { auto parquet_batch_reader = PrepareParquetFileBatchReader(file_name, schema_, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, batch_size); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), - std::numeric_limits::max()); ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( parquet_batch_reader.get())); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); parquet_batch_reader->Close(); auto expected_array = std::make_shared(struct_array_); ASSERT_TRUE(result_array->Equals(expected_array)); // test metrics auto read_metrics = parquet_batch_reader->GetReaderMetrics(); ASSERT_TRUE(read_metrics); - // TODO(jinli.zjw): test metrics - // ASSERT_TRUE(read_metrics->GetCounter(ParquetMetrics::READ_BYTES) > 0); - // ASSERT_TRUE(read_metrics->GetCounter(ParquetMetrics::READ_RAW_BYTES) > 0); + ASSERT_OK_AND_ASSIGN(uint64_t storage_read_bytes, + read_metrics->GetCounter(ParquetMetrics::READ_STORAGE_BYTES)); + ASSERT_GT(storage_read_bytes, 0u); } } @@ -338,6 +567,85 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchWithDictionary) { check_result(false); } +TEST_F(ParquetFileBatchReaderTest, TestNestedStructChildProjectionRecall) { + auto f0 = arrow::field("f0", arrow::int32()); + auto f1 = arrow::field( + "f1", arrow::struct_({arrow::field("c0", arrow::int64()), arrow::field("c1", arrow::utf8()), + arrow::field("c2", arrow::float64())})); + auto f2 = arrow::field("f2", arrow::utf8()); + + auto write_schema = arrow::schema({f0, f1, f2}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema->fields()), R"([ + [1, [100, "a", 1.1], "x"], + [2, [200, "b", 2.2], "y"], + [3, [300, null, 3.3], "z"] + ])") + .ValueOrDie()); + + WriteArray(file_path_, write_array, write_schema, + /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto read_schema = MakeReadSchema({ + f0, + arrow::field("f1", arrow::struct_({arrow::field("c1", arrow::utf8())})), + }); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(read_schema->fields()), {R"([ + [1, ["a"]], + [2, ["b"]], + [3, [null]] + ])"}, + &expected_array) + .ok()); + + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + +TEST_F(ParquetFileBatchReaderTest, TestReadSchemaWithMapSelectedKeysMetadata) { + auto id_field = arrow::field("id", arrow::int32()); + auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + + auto write_schema = arrow::schema({id_field, map_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema->fields()), R"([ + [1, [["k1", 10], ["k2", 20], ["k3", 30]]], + [2, [["k2", 200]]], + [3, null] + ])") + .ValueOrDie()); + + WriteArray(file_path_, write_array, write_schema, + /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + // selected-keys metadata is consumed by upper-level field mapping; format reader should + // still accept the schema and read data correctly. + auto read_schema = MakeReadSchema( + {id_field, WithMapSelectedKeys(map_field, "k1,k3")}); // NOLINT(whitespace/comma) + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + auto expected_array = arrow::ChunkedArray::Make({write_array}).ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + TEST_F(ParquetFileBatchReaderTest, TestGetFileSchemaWithFieldId) { std::string file_name = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" @@ -397,7 +705,9 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { ASSERT_EQ(arrow_reader_properties.batch_size(), 1024); ASSERT_EQ(arrow_reader_properties.use_threads(), true); ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 3); - ASSERT_EQ(arrow_reader_properties.cache_options(), arrow::io::CacheOptions::Defaults()); + auto expected_cache_options = arrow::io::CacheOptions::Defaults(); + expected_cache_options.hole_size_limit = DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT; + ASSERT_EQ(arrow_reader_properties.cache_options(), expected_cache_options); } { std::map options = {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}}; @@ -416,9 +726,43 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { ASSERT_EQ(arrow_reader_properties.use_threads(), true); ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 6); } + { + std::map options = { + {PARQUET_READ_CACHE_OPTION_LAZY, "true"}, + {PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT, "2"}, + {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "1048576"}, + {PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, "8388608"}, + }; + ASSERT_OK_AND_ASSIGN( + auto arrow_reader_properties, + ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024)); + const auto& cache_options = arrow_reader_properties.cache_options(); + ASSERT_TRUE(cache_options.lazy); + ASSERT_EQ(cache_options.prefetch_limit, 2); + ASSERT_EQ(cache_options.hole_size_limit, 1024 * 1024); + ASSERT_EQ(cache_options.range_size_limit, 8 * 1024 * 1024); + } + { + std::map options = { + {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "-1"}, + }; + ASSERT_NOK_WITH_MSG( + ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024), + "parquet.read.cache-option.hole-size-limit must be non-negative"); + } + { + std::map options = { + {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "1048576"}, + {PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, "1048576"}, + }; + ASSERT_NOK_WITH_MSG( + ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024), + "parquet.read.cache-option.range-size-limit must be greater than " + "parquet.read.cache-option.hole-size-limit"); + } } -TEST_F(ParquetFileBatchReaderTest, TestBitmapPushDownWithMultiRowGroups) { +TEST_F(ParquetFileBatchReaderTest, TestBitmapRowGroupPushDownWithMultiRowGroups) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; auto arrow_type = arrow::struct_(fields); auto src_array = std::dynamic_pointer_cast( @@ -456,8 +800,47 @@ TEST_F(ParquetFileBatchReaderTest, TestBitmapPushDownWithMultiRowGroups) { auto expected_array = arrow::ChunkedArray::Make({src_array->Slice(0, 6)}).ValueOrDie(); ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); } +TEST_F(ParquetFileBatchReaderTest, TestBitmapPagePushDownWithMultiRowGroups) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto arrow_type = arrow::struct_(fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([ + [0], + [1], + [2], + [3], + [4], + [5], + [6], + [7], + [8], + [9], + [10], + [11] + ])") + .ValueOrDie()); + auto src_schema = arrow::schema(fields); + std::optional bitmap = RoaringBitmap32::From({3, 5}); + // data in file rowGroup0:[0, 1, 2, 3, 4, 5] | rowGroup1:[6, 7, 8, 9, 10, 11] -TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPushDown) { + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/12, + /*enable_dictionary=*/true, + /*max_row_group_length=*/6); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, /*predicate=*/nullptr, bitmap, + /*batch_size=*/12, /*enable_page_level_filter=*/true); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + + auto expected_array = arrow::ChunkedArray(src_array->Slice(3, 3)); + ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); +} + +TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapRowGroupPushDown) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; auto arrow_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(arrow_type, arrow::default_memory_pool(), @@ -514,6 +897,65 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPushDown) { ASSERT_FALSE(result_array); } } +TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto arrow_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder(arrow_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto int_builder = static_cast(struct_builder.field_builder(0)); + int32_t length = 1024; + for (int32_t i = 0; i < length; ++i) { + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(int_builder->Append(i).ok()); + } + // data file: + // rowGroup0: [0, 256) + // rowGroup1: [256, 512) + // rowGroup2: [512, 768) + // rowGroup3: [768, 1024) + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_schema = arrow::schema(fields); + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1024, + /*enable_dictionary=*/true, + /*max_row_group_length=*/256); + { + // simple case + std::optional bitmap = RoaringBitmap32::From({100, 400, 600}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or( + {PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, + Literal(255)), + PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(600))})); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap, + /*batch_size=*/length, /*enable_page_level_filter=*/true); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + + auto expected_array = + arrow::ChunkedArray::Make({src_array->Slice(100, 1), src_array->Slice(600, 1)}) + .ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); + } + { + // test all data has been filtered out with predicate and bitmap pushdown + std::optional bitmap = RoaringBitmap32::From({100, 400, 600}); + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(800)); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap, + /*batch_size=*/length, /*enable_page_level_filter=*/true); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + ASSERT_FALSE(result_array); + } +} TEST_F(ParquetFileBatchReaderTest, TestReadNoField) { // if only read partition fields, format reader will set empty read schema @@ -526,20 +968,19 @@ TEST_F(ParquetFileBatchReaderTest, TestReadNoField) { PrepareParquetFileBatchReader(file_name, read_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); // read 2 rows - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), - std::numeric_limits::max()); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 0); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch2, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 2); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 2); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch3, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 4); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 4); // read rows with eof ASSERT_OK_AND_ASSIGN(auto batch4, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); parquet_batch_reader->Close(); arrow::FieldVector fields; @@ -643,4 +1084,241 @@ TEST_P(ParquetFileBatchReaderTest, TestTimestampType) { INSTANTIATE_TEST_SUITE_P(TestParam, ParquetFileBatchReaderTest, ::testing::Values(false, true)); +TEST_F(ParquetFileBatchReaderTest, TestAddMetadataPerFieldMetadata) { + // Write a simple parquet file, call AddMetadata to inject per-field metadata + // before Finish, then read back and verify the file schema carries the metadata. + auto write_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::float64()), + }); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), "update_schema_test.parquet"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path, /*overwrite=*/true)); + + ::parquet::WriterProperties::Builder builder; + builder.write_batch_size(10); + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN(auto format_writer, + ParquetFormatWriter::Create(out, write_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, pool_)); + + // Write one batch of data. + auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(write_schema->fields()), + R"([[1, "alice", 95.5], [2, "bob", 88.0], [3, "charlie", 72.3]])") + .ValueOrDie(); + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*data, &c_array).ok()); + ASSERT_OK(format_writer->AddBatch(&c_array)); + ASSERT_OK(format_writer->Flush()); + + // Build an updated schema with per-field metadata on "name" and "score". + auto name_meta = std::make_shared(); + name_meta->Append("shredding.field_mapping", "0:alice,1:bob,2:charlie"); + name_meta->Append("shredding.num_columns", "3"); + auto score_meta = std::make_shared(); + score_meta->Append("custom.unit", "percent"); + + auto updated_schema = arrow::schema({ + write_schema->field(0), // id — no metadata + write_schema->field(1)->WithMetadata(name_meta), // name — shredding metadata + write_schema->field(2)->WithMetadata(score_meta), // score — custom metadata + }); + + // AddMetadata must be called before Finish. + ASSERT_OK(format_writer->AddMetadata( + {{ArrowUtils::kArrowSchemaMetadataKey, SerializeSchemaToString(updated_schema)}})); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + + // Read back: GetFileSchema should reflect the updated per-field metadata. + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path, write_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Field 0 "id": no metadata. + ASSERT_EQ("id", file_schema->field(0)->name()); + + // Field 1 "name": should have the shredding metadata we set. + ASSERT_EQ("name", file_schema->field(1)->name()); + auto read_name_meta = file_schema->field(1)->metadata(); + ASSERT_NE(nullptr, read_name_meta); + auto field_mapping_val = read_name_meta->Get("shredding.field_mapping").ValueOrDie(); + ASSERT_EQ("0:alice,1:bob,2:charlie", field_mapping_val); + auto num_columns_val = read_name_meta->Get("shredding.num_columns").ValueOrDie(); + ASSERT_EQ("3", num_columns_val); + + // Field 2 "score": should have the custom metadata. + ASSERT_EQ("score", file_schema->field(2)->name()); + auto read_score_meta = file_schema->field(2)->metadata(); + ASSERT_NE(nullptr, read_score_meta); + auto unit_val = read_score_meta->Get("custom.unit").ValueOrDie(); + ASSERT_EQ("percent", unit_val); + + // Also verify data integrity — read it back and compare content. + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + ASSERT_EQ(result_array->num_chunks(), 1); + ASSERT_TRUE(data->Equals(*result_array->chunk(0))) << result_array->ToString(); +} + +TEST_F(ParquetFileBatchReaderTest, TestRowMappingSimple) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file rowGroup0:[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + // one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/12, /*max_page_size=*/1); + + // 1<=f0<=3 || 5<=f0<=6 + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(1), Literal(3)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(5), Literal(6))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2, + /*enable_page_level_filter=*/true); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch1 = src_array->Slice(1, 2); + ASSERT_TRUE(batch1->chunk(0)->Equals(expected_batch1)) << batch1->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 1); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 2); + // out of bound return invalid + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(2)); + + // Not adjacent pages + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[3], +[5] + ])") + .ValueOrDie()); + ASSERT_TRUE(batch2->chunk(0)->Equals(expected_batch2)) << batch2->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 5); + + // Only one record read + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch3, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch3 = src_array->Slice(6, 1); + ASSERT_TRUE(batch3->chunk(0)->Equals(expected_batch3)) << batch3->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(1)); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr eof_batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(nullptr, eof_batch); + // previous batch is eof, return invalid. + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); +} + +TEST_F(ParquetFileBatchReaderTest, TestRowMappingFullyAndPartially) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file RowGroup0:[0, 1, 2] | RowGroup1:[3, 4, 5] | RowGroup2:[6, 7, 8] | RowGroup3:[9, + // 10, 11] one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/3, /*max_page_size=*/1); + + // 3<=f0<=5 || f0==6 || f0==8 + // RowGroup 1 is fully matched, RowGroup 2 is partially matched, RowGroup 0 and RowGroup 3 are + // not matched. + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(3), Literal(5)), + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(6)), + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(8))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, + /*enable_page_level_filter=*/true); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(2).value(), 5); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 8); +} + +TEST_F(ParquetFileBatchReaderTest, TestRowMappingSetReadSchemaTwice) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file RowGroup0:[0, 1, 2] | RowGroup1:[3, 4, 5] | RowGroup2:[6, 7, 8] | RowGroup3:[9, + // 10, 11] one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/3, /*max_page_size=*/1); + + // 1<=f0<=3 || 6<=f0<=7 + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(1), Literal(3)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(6), Literal(7))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, + /*enable_page_level_filter=*/true); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 1); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 2); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(1)); + + ASSERT_OK_AND_ASSIGN( + predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(3), Literal(5))})); + + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, c_schema.get()).ok()); + ASSERT_OK( + parquet_batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/predicate, std::nullopt)); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch3, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(2).value(), 5); +} + } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index a1ae3473..433103e5 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -20,8 +20,23 @@ #include #include + +#include "fmt/format.h" +#include "paimon/status.h" + namespace paimon::parquet { +// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a +// Status. Used as the trailing catch clauses of a try block in every public +// method that calls into the parquet C++ API, so the read layer never throws. +#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ + catch (const std::exception& e) { \ + return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ + } \ + catch (...) { \ + return Status::UnknownError(fmt::format("{}: unknown error", (context))); \ + } + // write static inline const char PARQUET_BLOCK_SIZE[] = "parquet.block.size"; static inline const char PARQUET_PAGE_SIZE[] = "parquet.page.size"; @@ -39,6 +54,10 @@ static inline const char PARQUET_COMPRESSION_CODEC_BROTLI_LEVEL[] = "compression static inline const char PARQUET_WRITER_MAX_MEMORY_USE[] = "parquet.writer.max.memory.use"; static constexpr uint64_t DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE = 512 * 1024 * 1024; // 512MB +// Enable writing page index (ColumnIndex + OffsetIndex) for page-level filtering on read +static inline const char PARQUET_WRITE_ENABLE_PAGE_INDEX[] = "parquet.write.enable-page-index"; +static constexpr bool DEFAULT_PARQUET_WRITE_ENABLE_PAGE_INDEX = true; + // read static inline const char PARQUET_READ_EXECUTOR_THREAD_COUNT[] = "parquet.read.executor.thread-count"; @@ -46,20 +65,51 @@ static constexpr uint32_t DEFAULT_PARQUET_READ_EXECUTOR_THREAD_COUNT = 3; static inline const char PARQUET_READ_CACHE_OPTION_LAZY[] = "parquet.read.cache-option.lazy"; static inline const char PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT[] = "parquet.read.cache-option.prefetch-limit"; +// Arrow I/O range coalescing limits, both measured in bytes. Adjacent ranges whose gap is +// <= hole-size-limit can be merged; range-size-limit stops further merging once the combined +// span would exceed it. A single input range may itself be larger than range-size-limit. +static inline const char PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT[] = + "parquet.read.cache-option.hole-size-limit"; static inline const char PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT[] = "parquet.read.cache-option.range-size-limit"; +// Strategy for refining row ranges using the selection bitmap produced by pushed-down +// predicates. Two options: +// * "coalesce" (default): build row-level ranges from the bitmap, then merge nearby +// ranges whose gap is small enough (see PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT). +// * "trim": for each page with selected rows, trim its leading/trailing non-selected +// rows (and skip pages with no selected rows). Requires page index. Its advantage +// is stable, page-bounded ranges. After collected trimmed ranges for each column, +// the intersection of all columns' ranges is taken to produce the final row ranges. +static inline const char PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY[] = + "parquet.read.bitmap.row-range-refining-strategy"; +// When strategy = "coalesce", adjacent bitmap row ranges whose gap (in rows) is +// <= this limit are merged into one range; larger gaps are kept as real holes. +// A larger limit means fewer (larger) ranges and more wasted rows read; a smaller +// limit keeps the selection tighter at the cost of more (smaller) ranges. +// Only takes effect when PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY = "coalesce". +static inline const char PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT[] = + "parquet.read.bitmap.coalesce-hole-size-limit"; // stack-overflow may happen while the number of predicate node is too large, limit the number of // predicate nodes. Predicate will not be pushdown when exceed limit. static inline const char PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT[] = "parquet.read.predicate-node-count-limit"; +// Enable page-level filtering using column index +static inline const char PARQUET_READ_ENABLE_PAGE_INDEX_FILTER[] = + "parquet.read.enable-page-index-filter"; + // Default is true. Compaction will set to false to reduce memory consumption. static inline const char PARQUET_READ_ENABLE_PRE_BUFFER[] = "parquet.read.enable-pre-buffer"; static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT = 0; +// Default value of hole size limit, inherited from Arrow +static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT = 8 * 1024; static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT = 32 * 1024 * 1024; static constexpr uint32_t DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT = 512; +static constexpr bool DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER = true; +static constexpr char DEFAULT_PARQUET_READ_BITMAP_STRATEGY[] = "coalesce"; +static constexpr uint32_t DEFAULT_PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT = 32; class ParquetMetrics { public: @@ -71,6 +121,8 @@ class ParquetMetrics { "parquet.read.row-groups.after-filter"; static inline const char READ_ROWS[] = "parquet.read.rows"; static inline const char READ_BATCH_COUNT[] = "parquet.read.batch-count"; + // Byte-level read metric. storage-read-bytes: physical bytes read from storage. + static inline const char READ_STORAGE_BYTES[] = "parquet.read.storage-read-bytes"; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index 20079486..0a8e38b4 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -18,12 +18,16 @@ #include "paimon/format/parquet/parquet_format_writer.h" +#include #include +#include #include #include "arrow/c/bridge.h" #include "arrow/memory_pool.h" #include "arrow/record_batch.h" +#include "arrow/util/base64.h" +#include "arrow/util/key_value_metadata.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -81,6 +85,18 @@ Status ParquetFormatWriter::Finish() { return Status::OK(); } +Status ParquetFormatWriter::AddMetadata(const std::map& metadata) { + if (metadata.empty()) { + return Status::OK(); + } + auto key_value_metadata = std::make_shared(); + for (const auto& [key, value] : metadata) { + key_value_metadata->Append(key, arrow::util::base64_encode(std::string_view(value))); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(writer_->AddKeyValueMetadata(key_value_metadata)); + return Status::OK(); +} + Result ParquetFormatWriter::ReachTargetSize(bool suggested_check, int64_t target_size) const { if (suggested_check) { PAIMON_ASSIGN_OR_RAISE(const uint64_t length, GetEstimateLength()); diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h index e5cae163..4ab58d73 100644 --- a/src/paimon/format/parquet/parquet_format_writer.h +++ b/src/paimon/format/parquet/parquet_format_writer.h @@ -19,7 +19,9 @@ #pragma once #include +#include #include +#include #include "paimon/format/format_writer.h" #include "paimon/fs/file_system.h" @@ -64,6 +66,8 @@ class ParquetFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer, const std::shared_ptr& out, diff --git a/src/paimon/format/parquet/parquet_format_writer_test.cpp b/src/paimon/format/parquet/parquet_format_writer_test.cpp index ccd08680..f70c6332 100644 --- a/src/paimon/format/parquet/parquet_format_writer_test.cpp +++ b/src/paimon/format/parquet/parquet_format_writer_test.cpp @@ -31,6 +31,7 @@ #include "arrow/array/builder_primitive.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "arrow/io/file.h" #include "arrow/ipc/api.h" #include "arrow/memory_pool.h" diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h index c0d99315..51976372 100644 --- a/src/paimon/format/parquet/parquet_reader_builder.h +++ b/src/paimon/format/parquet/parquet_reader_builder.h @@ -18,18 +18,29 @@ #pragma once +#include +#include +#include #include #include #include #include +#include "arrow/buffer.h" +#include "arrow/io/memory.h" +#include "fmt/format.h" +#include "paimon/cache/cache.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/reader_builder.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/result.h" +#include "parquet/file_reader.h" +#include "parquet/file_writer.h" namespace paimon::parquet { @@ -43,20 +54,114 @@ class ParquetReaderBuilder : public ReaderBuilder { return this; } + ReaderBuilder* WithCache(const std::shared_ptr& cache) override { + cache_ = cache; + return this; + } + Result> Build( const std::shared_ptr& path) const override { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, path->Length()); - std::shared_ptr arrow_pool = GetArrowPool(pool_); - auto input_stream = - std::make_unique(path, arrow_pool, file_length); - return ParquetFileBatchReader::Create(std::move(input_stream), arrow_pool, options_, - batch_size_); + try { + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, path->Length()); + std::string file_uri; + if (cache_) { + Result file_uri_result = path->GetUri(); + if (file_uri_result.ok()) { + file_uri = std::move(file_uri_result).value(); + } + } + std::shared_ptr arrow_pool = GetArrowPool(pool_); + auto unique_input_stream = + std::make_unique(path, file_length, arrow_pool); + auto storage_read_bytes = unique_input_stream->StorageReadBytes(); + std::shared_ptr input_stream( + std::move(unique_input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<::parquet::FileMetaData> file_metadata, + GetCachedParquetMetadata(input_stream, file_uri, arrow_pool)); + return ParquetFileBatchReader::Create(std::move(input_stream), options_, batch_size_, + std::move(file_metadata), + std::move(storage_read_bytes), arrow_pool); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetReaderBuilder::Build") } private: + Result SerializeParquetMetadataFooter( + const std::shared_ptr& input_stream, + const ::parquet::ReaderProperties& reader_properties, + const std::shared_ptr& arrow_pool) const { + constexpr int64_t kParquetFooterSize = 8; + + std::shared_ptr<::parquet::FileMetaData> metadata = + ::parquet::ParquetFileReader::Open(input_stream, reader_properties)->metadata(); + if (metadata == nullptr) { + return Status::Invalid("Failed to read parquet metadata"); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr output_stream, + arrow::io::BufferOutputStream::Create(metadata->size() + kParquetFooterSize, + arrow_pool.get())); + ::parquet::WriteFileMetaData(*metadata, output_stream.get()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr metadata_footer, + output_stream->Finish()); + + MemorySegment segment = + MemorySegment::AllocateHeapMemory(metadata_footer->size(), pool_.get()); + std::memcpy(segment.MutableData(), metadata_footer->data(), metadata_footer->size()); + return segment; + } + + static Result> ParseParquetMetadataFooter( + const MemorySegment& segment, const ::parquet::ReaderProperties& reader_properties) { + if (segment.Data() == nullptr || segment.Size() <= 0) { + return Status::Invalid("Parquet metadata cache value is empty"); + } + + auto buffer = std::make_shared( + reinterpret_cast(segment.Data()), segment.Size()); + auto buffer_reader = std::make_shared(buffer); + std::shared_ptr<::parquet::FileMetaData> metadata = + ::parquet::ParquetFileReader::Open(buffer_reader, reader_properties)->metadata(); + if (metadata == nullptr) { + return Status::Invalid("Failed to parse parquet metadata footer"); + } + return metadata; + } + + Result> GetCachedParquetMetadata( + const std::shared_ptr& input_stream, + const std::string& file_uri, const std::shared_ptr& arrow_pool) const { + if (!cache_ || file_uri.empty()) { + return std::shared_ptr<::parquet::FileMetaData>(); + } + PAIMON_ASSIGN_OR_RAISE( + ::parquet::ReaderProperties reader_properties, + ParquetFileBatchReader::CreateReaderProperties(arrow_pool, options_)); + + auto cache_key = CacheKey::ForKind(file_uri, /*position=*/-1, /*length=*/-1, + CacheKind::DATA_FILE_FOOTER); + auto supplier = + [this, &input_stream, reader_properties, + arrow_pool](const std::shared_ptr&) -> Result> { + PAIMON_ASSIGN_OR_RAISE( + MemorySegment segment, + SerializeParquetMetadataFooter(input_stream, reader_properties, arrow_pool)); + return std::make_shared(segment, CacheCallback()); + }; + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache_value, + cache_->Get(cache_key, supplier)); + if (cache_value == nullptr) { + return Status::Invalid("Parquet metadata cache returned nullptr value"); + } + return ParseParquetMetadataFooter(cache_value->GetSegment(), reader_properties); + } + int32_t batch_size_ = -1; std::shared_ptr pool_; std::map options_; + std::shared_ptr cache_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index 8f8b82ad..f1df8b6e 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -275,10 +275,10 @@ ParquetStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fi const std::shared_ptr& pool) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, file_system->Open(path)); assert(input_stream); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); std::shared_ptr parquet_memory_pool = GetArrowPool(pool); auto parquet_input_file = std::make_shared( - std::move(input_stream), parquet_memory_pool, file_length); + std::move(input_stream), file_length, parquet_memory_pool); ::parquet::ReaderProperties read_properties(parquet_memory_pool.get()); read_properties.enable_buffered_stream(); ::parquet::arrow::FileReaderBuilder file_reader_builder; diff --git a/src/paimon/format/parquet/parquet_writer_builder.cpp b/src/paimon/format/parquet/parquet_writer_builder.cpp index 7fa18995..5946dd70 100644 --- a/src/paimon/format/parquet/parquet_writer_builder.cpp +++ b/src/paimon/format/parquet/parquet_writer_builder.cpp @@ -101,6 +101,15 @@ Result> ParquetWriterBuilder::Prepa PAIMON_ASSIGN_OR_RAISE(::parquet::ParquetVersion::type version, ConvertWriterVersion(writer_version)); builder.version(version); + + // Enable writing page index (ColumnIndex + OffsetIndex) for page-level filtering + PAIMON_ASSIGN_OR_RAISE(bool enable_page_index, OptionsUtils::GetValueFromMap( + options_, PARQUET_WRITE_ENABLE_PAGE_INDEX, + DEFAULT_PARQUET_WRITE_ENABLE_PAGE_INDEX)); + if (enable_page_index) { + builder.enable_write_page_index(); + } + return builder.build(); } diff --git a/src/paimon/format/parquet/predicate_converter.cpp b/src/paimon/format/parquet/predicate_converter.cpp index 819d2cc1..e81a675e 100644 --- a/src/paimon/format/parquet/predicate_converter.cpp +++ b/src/paimon/format/parquet/predicate_converter.cpp @@ -29,6 +29,7 @@ #include "fmt/format.h" #include "paimon/data/decimal.h" #include "paimon/defs.h" +#include "paimon/format/parquet/floating_point_predicate_utils.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/function.h" #include "paimon/predicate/leaf_predicate.h" @@ -36,6 +37,49 @@ #include "paimon/predicate/predicate.h" namespace paimon::parquet { +namespace { + +Result ConvertFloatingPointComparison( + const std::string& field_name, Function::Type function_type, const Literal& literal, + const arrow::compute::Expression& arrow_literal) { + arrow::compute::Expression field = arrow::compute::field_ref(field_name); + switch (function_type) { + case Function::Type::EQUAL: + return arrow::compute::equal(std::move(field), arrow_literal); + case Function::Type::NOT_EQUAL: + if (FloatingPointPredicateUtils::IsZero(literal)) { + // Arrow treats both zero signs as equal. Keep every non-null row group so the + // exact predicate can distinguish the opposite zero sign. + return arrow::compute::is_valid(std::move(field)); + } + return arrow::compute::not_equal(std::move(field), arrow_literal); + case Function::Type::GREATER_THAN: + if (FloatingPointPredicateUtils::IsNegativeZero(literal)) { + // parquet-mr normalizes a zero upper bound to +0.0. Arrow treats both zero signs + // as equal, so >= is the safe pruning equivalent of Java's > -0.0. + return arrow::compute::greater_equal(std::move(field), arrow_literal); + } + return arrow::compute::greater(std::move(field), arrow_literal); + case Function::Type::GREATER_OR_EQUAL: + return arrow::compute::greater_equal(std::move(field), arrow_literal); + case Function::Type::LESS_THAN: + if (FloatingPointPredicateUtils::IsZero(literal) && + !FloatingPointPredicateUtils::IsNegativeZero(literal)) { + // parquet-mr normalizes a zero lower bound to -0.0. Arrow's <= 0.0 retains the + // page for Java's -0.0 < +0.0. + return arrow::compute::less_equal(std::move(field), arrow_literal); + } + return arrow::compute::less(std::move(field), arrow_literal); + case Function::Type::LESS_OR_EQUAL: + return arrow::compute::less_equal(std::move(field), arrow_literal); + default: + return Status::Invalid(fmt::format("invalid floating-point comparison type {}", + static_cast(function_type))); + } +} + +} // namespace + arrow::compute::Expression PredicateConverter::AlwaysTrue() { static const arrow::compute::Expression expr = arrow::compute::literal(true); return expr; @@ -143,6 +187,10 @@ Result PredicateConverter::ConvertLeaf( const auto& literals = leaf_predicate->Literals(); const auto& function = leaf_predicate->GetFunction(); auto function_type = function.GetType(); + if (FloatingPointPredicateUtils::IsType(leaf_predicate->GetFieldType()) && + FloatingPointPredicateUtils::IsComparison(function_type)) { + return ConvertFloatingPointLeaf(leaf_predicate); + } switch (function_type) { case Function::Type::IS_NULL: { return arrow::compute::is_null(arrow::compute::field_ref(field_name), @@ -244,6 +292,34 @@ Result PredicateConverter::ConvertLeaf( return Status::OK(); } +Result PredicateConverter::ConvertFloatingPointLeaf( + const std::shared_ptr& leaf_predicate) { + const auto& field_name = leaf_predicate->FieldName(); + const auto& literals = leaf_predicate->Literals(); + const auto& function = leaf_predicate->GetFunction(); + Function::Type function_type = function.GetType(); + PAIMON_RETURN_NOT_OK(CheckLiteralNotEmpty(literals, function, field_name)); + + if (function_type == Function::Type::IN || function_type == Function::Type::NOT_IN) { + Function::Type comparison_type = + function_type == Function::Type::IN ? Function::Type::EQUAL : Function::Type::NOT_EQUAL; + std::vector sub_exprs; + sub_exprs.reserve(literals.size()); + for (const auto& literal : literals) { + CONVERT_TO_ARROW_LITERAL(literal); + PAIMON_ASSIGN_OR_RAISE(arrow::compute::Expression sub_expr, + ConvertFloatingPointComparison(field_name, comparison_type, + literal, arrow_literal)); + sub_exprs.push_back(std::move(sub_expr)); + } + return function_type == Function::Type::IN ? arrow::compute::or_(sub_exprs) + : arrow::compute::and_(sub_exprs); + } + + CONVERT_TO_ARROW_LITERAL(literals[0]); + return ConvertFloatingPointComparison(field_name, function_type, literals[0], arrow_literal); +} + Result PredicateConverter::ConvertToArrowLiteral( const Literal& literal) { auto literal_type = literal.GetType(); diff --git a/src/paimon/format/parquet/predicate_converter.h b/src/paimon/format/parquet/predicate_converter.h index 5b44f846..e4df06be 100644 --- a/src/paimon/format/parquet/predicate_converter.h +++ b/src/paimon/format/parquet/predicate_converter.h @@ -68,6 +68,9 @@ class PredicateConverter { static Result ConvertLeaf( const std::shared_ptr& leaf_predicate); + static Result ConvertFloatingPointLeaf( + const std::shared_ptr& leaf_predicate); + static Result ConvertToArrowLiteral(const Literal& literal); }; diff --git a/src/paimon/format/parquet/predicate_converter_test.cpp b/src/paimon/format/parquet/predicate_converter_test.cpp index a6661dc3..542ea0f4 100644 --- a/src/paimon/format/parquet/predicate_converter_test.cpp +++ b/src/paimon/format/parquet/predicate_converter_test.cpp @@ -191,6 +191,128 @@ TEST(PredicateConverterTest, TestSimple) { } } +TEST(PredicateConverterTest, TestJavaCompatibleSignedZeroExpressions) { + struct TestType { + FieldType field_type; + Literal negative_zero; + Literal positive_zero; + Literal one; + }; + const std::vector test_types = { + {FieldType::FLOAT, Literal(-0.0f), Literal(0.0f), Literal(1.0f)}, + {FieldType::DOUBLE, Literal(-0.0), Literal(0.0), Literal(1.0)}}; + + for (const auto& test_type : test_types) { + auto equal_negative_zero = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto equal_negative_zero_expr, + PredicateConverter::Convert(equal_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 == -0)", equal_negative_zero_expr.ToString()); + + auto equal_positive_zero = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto equal_positive_zero_expr, + PredicateConverter::Convert(equal_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 == 0)", equal_positive_zero_expr.ToString()); + + auto greater_negative_zero = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto greater_negative_zero_expr, + PredicateConverter::Convert(greater_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 >= -0)", greater_negative_zero_expr.ToString()); + + auto less_positive_zero = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto less_positive_zero_expr, + PredicateConverter::Convert(less_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 <= 0)", less_positive_zero_expr.ToString()); + + auto not_equal_negative_zero = PredicateBuilder::NotEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto not_equal_negative_zero_expr, + PredicateConverter::Convert(not_equal_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("is_valid(f0)", not_equal_negative_zero_expr.ToString()); + + auto not_equal_positive_zero = PredicateBuilder::NotEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto not_equal_positive_zero_expr, + PredicateConverter::Convert(not_equal_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("is_valid(f0)", not_equal_positive_zero_expr.ToString()); + + auto greater_positive_zero = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto greater_positive_zero_expr, + PredicateConverter::Convert(greater_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 > 0)", greater_positive_zero_expr.ToString()); + + auto greater_or_equal_positive_zero = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto greater_or_equal_positive_zero_expr, + PredicateConverter::Convert(greater_or_equal_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 >= 0)", greater_or_equal_positive_zero_expr.ToString()); + + auto greater_or_equal_negative_zero = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto greater_or_equal_negative_zero_expr, + PredicateConverter::Convert(greater_or_equal_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 >= -0)", greater_or_equal_negative_zero_expr.ToString()); + + auto less_negative_zero = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto less_negative_zero_expr, + PredicateConverter::Convert(less_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 < -0)", less_negative_zero_expr.ToString()); + + auto less_or_equal_negative_zero = PredicateBuilder::LessOrEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero); + ASSERT_OK_AND_ASSIGN(auto less_or_equal_negative_zero_expr, + PredicateConverter::Convert(less_or_equal_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 <= -0)", less_or_equal_negative_zero_expr.ToString()); + + auto less_or_equal_positive_zero = PredicateBuilder::LessOrEqual( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero); + ASSERT_OK_AND_ASSIGN(auto less_or_equal_positive_zero_expr, + PredicateConverter::Convert(less_or_equal_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(f0 <= 0)", less_or_equal_positive_zero_expr.ToString()); + + auto in_negative_zero = PredicateBuilder::In( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, + {test_type.negative_zero, test_type.one}); + ASSERT_OK_AND_ASSIGN(auto in_negative_zero_expr, + PredicateConverter::Convert(in_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("((f0 == -0) or (f0 == 1))", in_negative_zero_expr.ToString()); + + auto in_positive_zero = PredicateBuilder::In( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, + {test_type.positive_zero, test_type.one}); + ASSERT_OK_AND_ASSIGN(auto in_positive_zero_expr, + PredicateConverter::Convert(in_positive_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("((f0 == 0) or (f0 == 1))", in_positive_zero_expr.ToString()); + + auto not_in_negative_zero = PredicateBuilder::NotIn( + /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, + {test_type.negative_zero, test_type.one}); + ASSERT_OK_AND_ASSIGN(auto not_in_negative_zero_expr, + PredicateConverter::Convert(not_in_negative_zero, + /*predicate_node_count_limit=*/100)); + ASSERT_EQ("(is_valid(f0) and (f0 != 1))", not_in_negative_zero_expr.ToString()); + } +} + TEST(PredicateConverterTest, TestCompound) { // "struct"; { @@ -387,9 +509,9 @@ TEST(PredicateConverterTest, TestWithoutPredicate) { } TEST(PredicateConverterTest, TestInvalidCase) { - auto predicate = + auto empty_in = PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, {}); - ASSERT_NOK_WITH_MSG(PredicateConverter::Convert(predicate, /*predicate_node_count_limit=*/100), + ASSERT_NOK_WITH_MSG(PredicateConverter::Convert(empty_in, /*predicate_node_count_limit=*/100), "predicate [In] need literal on field f0"); } diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp index 7ccbaa2c..3d169db3 100644 --- a/src/paimon/format/parquet/predicate_pushdown_test.cpp +++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp @@ -81,7 +81,8 @@ class PredicatePushdownTest : public ::testing::Test { void TearDown() override {} - void PrepareTestData(const std::shared_ptr& struct_array) { + void PrepareTestData(const std::shared_ptr& struct_array, + bool enable_page_index = false) { auto data_type = struct_array->struct_type(); auto data_schema = arrow::schema(data_type->fields()); auto data_arrow_array = std::make_unique(); @@ -90,6 +91,10 @@ class PredicatePushdownTest : public ::testing::Test { fs_->Create(file_name_, /*overwrite=*/false)); ::parquet::WriterProperties::Builder builder; builder.write_batch_size(batch_size_); + if (enable_page_index) { + builder.enable_write_page_index(); + builder.data_pagesize(1); + } auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN( auto format_writer, @@ -106,15 +111,16 @@ class PredicatePushdownTest : public ::testing::Test { uint32_t predicate_node_count_limit = paimon::parquet::DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); std::map options; options[paimon::parquet::PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT] = std::to_string(predicate_node_count_limit); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), arrow_pool_, - options, batch_size_)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size_, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); ASSERT_TRUE(arrow_status.ok()); @@ -426,6 +432,48 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithAllDataNull) { } } +TEST_F(PredicatePushdownTest, TestSignedZeroRowGroupPruning) { + auto value_field = arrow::field("value", arrow::float64()); + std::shared_ptr data = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}), + R"([[-0.0], [0.0]])") + .ValueOrDie()); + PrepareTestData(data); + + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(-0.0)); + CheckResult(arrow::schema({value_field}), predicate, data); +} + +TEST_F(PredicatePushdownTest, TestFiniteGreaterThanStillPrunesRowGroup) { + auto value_field = arrow::field("value", arrow::float64()); + std::shared_ptr data = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}), R"([[0.5]])") + .ValueOrDie()); + PrepareTestData(data); + + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(1.0)); + CheckResult(arrow::schema({value_field}), predicate, /*expected_array=*/nullptr); +} + +TEST_F(PredicatePushdownTest, TestSignedZeroPageIndexPruning) { + auto value_field = arrow::field("value", arrow::float64()); + std::shared_ptr data = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}), R"([ + [-0.0], [-0.0], [-0.0], [-0.0], [-0.0], + [-0.0], [-0.0], [-0.0], [-0.0], [-0.0], + [1.0], [1.0], [1.0], [1.0], [1.0], + [1.0], [1.0], [1.0], [1.0], [1.0] + ])") + .ValueOrDie()); + PrepareTestData(data, /*enable_page_index=*/true); + + auto predicate = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(0.0)); + CheckResult(arrow::schema({value_field}), predicate, data->Slice(0, batch_size_)); +} + TEST_F(PredicatePushdownTest, TestCompoundPredicate) { PrepareTestData(struct_array_); auto read_schema = arrow::schema(struct_array_->struct_type()->fields()); diff --git a/src/paimon/format/parquet/row_ranges.cpp b/src/paimon/format/parquet/row_ranges.cpp new file mode 100644 index 00000000..2bf469e1 --- /dev/null +++ b/src/paimon/format/parquet/row_ranges.cpp @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/row_ranges.h" + +#include +#include + +namespace paimon::parquet { + +namespace { + +// Returns the union of the two ranges or nullopt if there are elements between them. +// Used by Add to splice an inserted range into the existing sorted-disjoint sequence. +std::optional UnionRanges(const RowRanges::Range& left, + const RowRanges::Range& right) { + if (left.from <= right.from) { + if (left.to + 1 >= right.from) { + return RowRanges::Range(left.from, std::max(left.to, right.to)); + } + } else if (right.to + 1 >= left.from) { + return RowRanges::Range(right.from, std::max(left.to, right.to)); + } + return std::nullopt; +} + +} // namespace + +RowRanges RowRanges::Union(const RowRanges& left, const RowRanges& right) { + std::vector combined; + combined.reserve(left.ranges_.size() + right.ranges_.size()); + combined.insert(combined.end(), left.ranges_.begin(), left.ranges_.end()); + combined.insert(combined.end(), right.ranges_.begin(), right.ranges_.end()); + return RowRanges(Range::SortAndMergeOverlap(combined, /*adjacent=*/true)); +} + +RowRanges RowRanges::Intersection(const RowRanges& left, const RowRanges& right) { + return RowRanges(Range::And(left.ranges_, right.ranges_)); +} + +int64_t RowRanges::RowCount() const { + int64_t count = 0; + for (const auto& range : ranges_) { + count += range.Count(); + } + return count; +} + +bool RowRanges::IsOverlapping(int64_t from, int64_t to) const { + Range target(from, to); + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), target, + [](const Range& r, const Range& t) { return r.to < t.from; }); + return it != ranges_.end() && it->from <= target.to; +} + +void RowRanges::Add(const Range& range) { + if (ranges_.empty()) { + ranges_.push_back(range); + return; + } + + // Find insertion point using binary search (sorted by 'from') + auto pos = + std::lower_bound(ranges_.begin(), ranges_.end(), range, + [](const Range& r, const Range& target) { return r.from < target.from; }); + + // Scan backward and forward to find all ranges that overlap or are adjacent + Range merged = range; + auto merge_begin = pos; + auto merge_end = pos; + + // Merge with preceding ranges + while (merge_begin != ranges_.begin()) { + auto prev = merge_begin - 1; + auto u = UnionRanges(*prev, merged); + if (!u.has_value()) break; + merged = u.value(); + merge_begin = prev; + } + + // Merge with following ranges + while (merge_end != ranges_.end()) { + auto u = UnionRanges(*merge_end, merged); + if (!u.has_value()) break; + merged = u.value(); + ++merge_end; + } + + // Replace [merge_begin, merge_end) with the single merged range + auto it = ranges_.erase(merge_begin, merge_end); + ranges_.insert(it, merged); +} + +std::optional RowRanges::MapFilteredIndexToOriginalRow(int64_t filtered_index) const { + int64_t accumulated = 0; + for (const auto& range : ranges_) { + int64_t count = range.Count(); + if (filtered_index < accumulated + count) { + return range.from + (filtered_index - accumulated); + } + accumulated += count; + } + return std::nullopt; +} + +std::string RowRanges::ToString() const { + if (ranges_.empty()) { + return "[]"; + } + std::string result = "["; + for (size_t i = 0; i < ranges_.size(); ++i) { + if (i > 0) { + result += ", "; + } + result += ranges_[i].ToString(); + } + result += "]"; + return result; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h new file mode 100644 index 00000000..51fea472 --- /dev/null +++ b/src/paimon/format/parquet/row_ranges.h @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/utils/range.h" + +namespace paimon::parquet { + +/// RowRanges represents a set of row ranges in a row group. +/// Each range is defined by [from, to] where both are inclusive. +/// This is used for page-level filtering to skip rows that don't match predicates. +class RowRanges { + public: + /// A single inclusive range. Aliased to paimon::Range so the parquet code shares the + /// common range type and helpers (Intersection, And, SortAndMergeOverlap, ...). + using Range = paimon::Range; + + /// Creates an empty RowRanges. + RowRanges() = default; + + /// Creates a RowRanges with a single range [from, to]. + explicit RowRanges(const Range& range) : ranges_({range}) {} + + /// Creates a RowRanges from a list of ranges. + explicit RowRanges(const std::vector& ranges) : ranges_(ranges) {} + + /// Creates a RowRanges from a list of ranges, taking ownership of the vector. + explicit RowRanges(std::vector&& ranges) : ranges_(std::move(ranges)) {} + + /// Creates a RowRanges with a single range [0, row_count - 1]. + static RowRanges CreateSingle(int64_t row_count) { + if (row_count <= 0) { + return RowRanges(); + } + return RowRanges(Range(0, row_count - 1)); + } + + /// Creates an empty RowRanges. + static RowRanges CreateEmpty() { + return RowRanges(); + } + + /// Calculates the union of two RowRanges. + /// The union contains all row indexes that were contained in either of the inputs. + static RowRanges Union(const RowRanges& left, const RowRanges& right); + + /// Calculates the intersection of two RowRanges. + /// The intersection contains all row indexes that were contained in both inputs. + static RowRanges Intersection(const RowRanges& left, const RowRanges& right); + + /// Returns the number of rows in the ranges. + int64_t RowCount() const; + + /// Returns the ranges. + const std::vector& GetRanges() const { + return ranges_; + } + + /// Returns true if there are no ranges. + bool IsEmpty() const { + return ranges_.empty(); + } + + /// Returns true if the specified range overlaps with any of the ranges. + bool IsOverlapping(int64_t from, int64_t to) const; + + /// Returns true if the specified row is contained in any of the ranges. + bool Contains(int64_t row) const { + return IsOverlapping(row, row); + } + + /// Adds a range to the end of the list, maintaining sorted disjoint ranges. + void Add(const Range& range); + + /// Maps a filtered-result index to the original row index within the row group. + /// For example, if RowRanges = {[10,19], [50,59]}, then: + /// MapFilteredIndexToOriginalRow(0) = 10 (first row of first range) + /// MapFilteredIndexToOriginalRow(9) = 19 (last row of first range) + /// MapFilteredIndexToOriginalRow(10) = 50 (first row of second range) + /// Returns nullopt if filtered_index is out of bounds. + std::optional MapFilteredIndexToOriginalRow(int64_t filtered_index) const; + + std::string ToString() const; + + private: + std::vector ranges_; +}; +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/target_row_group.h b/src/paimon/format/parquet/target_row_group.h new file mode 100644 index 00000000..b86e29df --- /dev/null +++ b/src/paimon/format/parquet/target_row_group.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/format/parquet/row_ranges.h" + +namespace paimon::parquet { +class TargetRowGroup; +using TargetRowGroups = std::vector; +class TargetRowGroup { + public: + TargetRowGroup(int32_t rg_index, bool is_partially_matched, RowRanges ranges) + : row_group_index_(rg_index), + is_partially_matched_(is_partially_matched), + row_ranges_(std::move(ranges)) {} + + TargetRowGroup(const TargetRowGroup& other) = default; + TargetRowGroup& operator=(const TargetRowGroup& other) = default; + + bool IsExcludedByReadRange() const { + return excluded_by_read_range_; + } + + void SetExcludedByReadRange(bool excluded) { + excluded_by_read_range_ = excluded; + } + + int32_t GetRowGroupIndex() const { + return row_group_index_; + } + + bool IsPartiallyMatched() const { + return is_partially_matched_; + } + + const RowRanges& GetRowRanges() const { + return row_ranges_; + } + + // Create a list of TargetRowGroups for serial (non-filtered) reading. + // + // Each element in 'ranges' is a (start, end) pair describing the row + // range of a single row group. The vector index 'i' is used as the row + // group index, so the caller must ensure that 'ranges' is ordered to + // match the physical row-group order in the file. + // + // For each valid range (start < end), a TargetRowGroup is created with: + // - row_group_index = i + // - is_partially_matched = false (the entire row group is read) + // - row_ranges = [0, end - start - 1] (local row indices covering the + // full group; converted from absolute offsets to 0-based local offsets) + // + // Ranges where start >= end are treated as empty and skipped. + static TargetRowGroups MakeForAllRowGroups( + const std::vector>& ranges) { + TargetRowGroups target_row_groups; + target_row_groups.reserve(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + // Skip empty or invalid ranges. + if (ranges[i].first >= ranges[i].second) { + continue; + } + // Convert the absolute [start, end) pair into a 0-based local + // row range [0, row_count - 1] for this row group. + target_row_groups.emplace_back( + static_cast(i), false, + RowRanges(Range(0, ranges[i].second - ranges[i].first - 1))); + } + return target_row_groups; + } + + static std::vector GetRowGroupIndices(const TargetRowGroups& target_row_groups) { + std::vector indices; + indices.reserve(target_row_groups.size()); + for (const auto& rg : target_row_groups) { + indices.push_back(rg.GetRowGroupIndex()); + } + return indices; + } + + private: + int32_t row_group_index_{-1}; + bool is_partially_matched_{false}; + // Local row ranges + RowRanges row_ranges_; + // Whether this row group has been excluded by ApplyReadRanges. + // When true, this row group is logically skipped during iteration + // but retained so that a subsequent wider ApplyReadRanges can restore it. + bool excluded_by_read_range_{false}; +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp new file mode 100644 index 00000000..b2235b2a --- /dev/null +++ b/src/paimon/format/parquet/variant_parquet_test.cpp @@ -0,0 +1,1171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/io/file.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/shredding_file_reader.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/core_options.h" +#include "paimon/data/variant.h" +#include "paimon/format/parquet/parquet_field_id_converter.h" +#include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#include "parquet/metadata.h" +#include "parquet/properties.h" +#include "parquet/schema.h" + +namespace paimon::parquet::test { + +class VariantParquetTest : public ::testing::Test { + public: + void SetUp() override { + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = std::make_shared(); + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + file_path_ = PathUtil::JoinPath(dir_->Str(), "variant.parquet"); + + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + paimon_schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + + // `[id, s: struct]`: a variant nested inside a ROW column, next to + // a plain sibling that must survive the physical substitution and the reassembly. + nested_sibling_field_ = arrow::field("t", arrow::utf8()); + std::vector nested_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("s", arrow::struct_({VariantTypeUtils::ToArrowField("nv"), + nested_sibling_field_})))}; + nested_schema_ = DataField::ConvertDataFieldsToArrowSchema(nested_fields); + nested_variant_field_ = nested_schema_->field(1)->type()->field(0); + + // `[id, arr: ARRAY]` and `[id, m: MAP]`: variants inside + // repeated groups, which are never shredded but can still be read as a projection. + list_element_field_ = VariantTypeUtils::ToArrowField("element"); + std::vector list_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("arr", arrow::list(list_element_field_)))}; + list_schema_ = DataField::ConvertDataFieldsToArrowSchema(list_fields); + list_element_field_ = list_schema_->field(1)->type()->field(0); + + map_item_field_ = VariantTypeUtils::ToArrowField("value"); + std::vector map_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("m", arrow::map(arrow::utf8(), map_item_field_)))}; + map_schema_ = DataField::ConvertDataFieldsToArrowSchema(map_fields); + map_item_field_ = map_schema_->field(1)->type()->field(0)->type()->field(1); + + // `[id, arr2: ARRAY>]`: a variant one struct level below a + // repeated group, where the read and file children must line up field by field. + std::vector list_struct_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField( + 2, arrow::field("arr2", + arrow::list(arrow::field( + "element", arrow::struct_({VariantTypeUtils::ToArrowField("v"), + nested_sibling_field_})))))}; + list_struct_schema_ = DataField::ConvertDataFieldsToArrowSchema(list_struct_fields); + list_struct_variant_field_ = + list_struct_schema_->field(1)->type()->field(0)->type()->field(0); + } + + std::shared_ptr BuildArray(const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr batch, + paimon::test::VariantTestData::BuildVariantBatch( + paimon_schema_->field(0), paimon_schema_->field(1), jsons, pool_)); + return batch; + } + + // Writes one batch with the given logical schema through the production parquet write path + // (mapping paimon field ids to parquet field ids). + void WriteFile(const std::shared_ptr& schema, ArrowArray* c_array) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr write_schema, + ParquetFieldIdConverter::AddParquetIdsFromPaimonIds(schema)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path_, /*overwrite=*/true)); + ::parquet::WriterProperties::Builder builder; + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, write_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(c_array)); + ASSERT_OK(format_writer->Flush()); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + } + + void WriteFile(const std::shared_ptr& array) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*array, arrow_array.get()).ok()); + WriteFile(paimon_schema_, arrow_array.get()); + } + + void WriteShreddedFile(const std::vector& jsons, + const std::shared_ptr& plan) { + ASSERT_NE(plan, nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + auto logical = BuildArray(jsons); + auto c_logical = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*logical, c_logical.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_logical.get())); + WriteFile(converter->GetPhysicalSchema(), c_physical.get()); + } + + // Writes `jsons` shredded according to the configured ROW-type shredding schema JSON. + void WriteShreddedFile(const std::vector& jsons, + const char* shredding_schema_json) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + VariantShreddingWritePlan::FromConfiguredSchema(paimon_schema_, shredding_schema_json)); + WriteShreddedFile(jsons, plan); + } + + // Writes `jsons` using the given inferred shredding type for the top-level Variant column. + void WriteShreddedFile(const std::vector& jsons, + const std::shared_ptr& shredding_type) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + VariantShreddingWritePlan::Create(paimon_schema_, {{"v", shredding_type}})); + WriteShreddedFile(jsons, plan); + } + + static std::string NestedSiblingValue(size_t row) { + return "t" + std::to_string(row); + } + + std::shared_ptr BuildIdColumn(size_t rows) { + arrow::Int32Builder id_builder; + for (size_t i = 0; i < rows; ++i) { + EXPECT_TRUE(id_builder.Append(static_cast(i)).ok()); + } + std::shared_ptr ids; + EXPECT_TRUE(id_builder.Finish(&ids).ok()); + return ids->data(); + } + + // Builds the offsets buffer of a repeated column, flattening its elements into `flat`. + std::shared_ptr BuildOffsetsBuffer( + const std::vector>& rows, std::vector* flat) { + arrow::Int32Builder offset_builder; + for (const auto& row : rows) { + EXPECT_TRUE(offset_builder.Append(static_cast(flat->size())).ok()); + flat->insert(flat->end(), row.begin(), row.end()); + } + EXPECT_TRUE(offset_builder.Append(static_cast(flat->size())).ok()); + std::shared_ptr offsets; + EXPECT_TRUE(offset_builder.Finish(&offsets).ok()); + return offsets->data()->buffers[1]; + } + + // Builds a `[id, arr: list]` batch: row `i` holds the variants of `rows[i]`. + // + // The repeated columns here are assembled at the `ArrayData` level on purpose: Arrow's + // `FromArrays` helpers `checked_cast` their arguments, which is a `dynamic_cast` in a debug + // build and fails across the test binary / libpaimon boundary for templated array classes. + std::shared_ptr BuildListArray( + const std::vector>& rows) { + std::vector flat; + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + // The element variants are built as one flat batch that the offsets slice into rows. + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + list_schema_->field(0), list_element_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + auto list_data = arrow::ArrayData::Make( + list_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {elements.value()->field(1)->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(list_schema_->fields()), static_cast(rows.size()), {nullptr}, + {BuildIdColumn(rows.size()), list_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `arr: list` column. + void WriteListFile(const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildListArray(rows), arrow_array.get()).ok()); + WriteFile(list_schema_, arrow_array.get()); + } + + // Builds a `[id, m: map]` batch: row `i` maps `keys[i][k]` to the variant of + // `rows[i][k]`. + std::shared_ptr BuildMapArray( + const std::vector>& keys, + const std::vector>& rows) { + std::vector flat; + arrow::StringBuilder key_builder; + EXPECT_EQ(keys.size(), rows.size()); + for (size_t i = 0; i < keys.size() && i < rows.size(); ++i) { + EXPECT_EQ(keys[i].size(), rows[i].size()) << "row " << i; + for (const auto& key : keys[i]) { + EXPECT_TRUE(key_builder.Append(key).ok()); + } + } + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + std::shared_ptr map_keys; + EXPECT_TRUE(key_builder.Finish(&map_keys).ok()); + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + map_schema_->field(0), map_item_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + // A map array is a list of `struct` entries. + auto entries_data = arrow::ArrayData::Make( + map_schema_->field(1)->type()->field(0)->type(), static_cast(flat.size()), + {nullptr}, {map_keys->data(), elements.value()->field(1)->data()}, /*null_count=*/0); + auto map_data = + arrow::ArrayData::Make(map_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {entries_data}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(map_schema_->fields()), static_cast(rows.size()), {nullptr}, + {BuildIdColumn(rows.size()), map_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `m: map` column. + void WriteMapFile(const std::vector>& keys, + const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildMapArray(keys, rows), arrow_array.get()).ok()); + WriteFile(map_schema_, arrow_array.get()); + } + + // Builds a `[id, arr2: list>]` batch. The sibling of the + // `k`-th element over the whole column is `NestedSiblingValue(k)`. + std::shared_ptr BuildListStructArray( + const std::vector>& rows) { + std::vector flat; + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + list_struct_schema_->field(0), list_struct_variant_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < flat.size(); ++i) { + EXPECT_TRUE(sibling_builder.Append(NestedSiblingValue(i)).ok()); + } + std::shared_ptr sibling; + EXPECT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto element_data = arrow::ArrayData::Make( + list_struct_schema_->field(1)->type()->field(0)->type(), + static_cast(flat.size()), {nullptr}, + {elements.value()->field(1)->data(), sibling->data()}, /*null_count=*/0); + auto list_data = arrow::ArrayData::Make( + list_struct_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {element_data}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(list_struct_schema_->fields()), static_cast(rows.size()), + {nullptr}, {BuildIdColumn(rows.size()), list_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `arr2: list>` column. + void WriteListStructFile(const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildListStructArray(rows), arrow_array.get()).ok()); + WriteFile(list_struct_schema_, arrow_array.get()); + } + + // Builds a `[id, s: struct]` batch holding the variant encodings of `jsons`. + std::shared_ptr BuildNestedArray(const std::vector& jsons) { + auto batch = paimon::test::VariantTestData::BuildVariantBatch( + nested_schema_->field(0), nested_variant_field_, jsons, pool_); + EXPECT_TRUE(batch.ok()) << batch.status().ToString(); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < jsons.size(); ++i) { + EXPECT_TRUE(sibling_builder.Append(NestedSiblingValue(i)).ok()); + } + std::shared_ptr sibling; + EXPECT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto struct_column = arrow::StructArray::Make( + {batch.value()->field(1), sibling}, {nested_variant_field_, nested_sibling_field_}); + EXPECT_TRUE(struct_column.ok()) << struct_column.status().ToString(); + auto nested = + arrow::StructArray::Make({batch.value()->field(0), struct_column.ValueOrDie()}, + {nested_schema_->field(0), nested_schema_->field(1)}); + EXPECT_TRUE(nested.ok()) << nested.status().ToString(); + return nested.ValueOrDie(); + } + + // Writes `jsons` into `s.nv` unshredded. + void WriteNestedFile(const std::vector& jsons) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildNestedArray(jsons), arrow_array.get()).ok()); + WriteFile(nested_schema_, arrow_array.get()); + } + + // Writes `jsons` into `s.nv` shredded by `shredding_type` (the nested variant is addressed by + // its field-index path `{1, 0}`). + void WriteShreddedNestedFile(const std::vector& jsons, + const std::shared_ptr& shredding_type) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + VariantShreddingWritePlan::CreateFromPaths( + nested_schema_, {{std::vector{1, 0}, shredding_type}})); + ASSERT_NE(plan, nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + auto c_logical = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildNestedArray(jsons), c_logical.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_logical.get())); + WriteFile(converter->GetPhysicalSchema(), c_physical.get()); + } + + // Builds the read schema projecting `s.nv` as the given variant-access projection, keeping + // the plain sibling `s.t`. + std::shared_ptr BuildNestedAccessReadSchema( + const std::vector, std::string>>& accesses) { + auto access_field = BuildAccessField(accesses, "nv"); + auto read_struct = nested_schema_->field(1)->WithType( + arrow::struct_({access_field, nested_sibling_field_})); + return arrow::schema({nested_schema_->field(0), read_struct}); + } + + // Asserts that the plain sibling column of `s` round-tripped unchanged. + void ExpectNestedSibling(const std::shared_ptr& s_column) { + const auto& sibling = static_cast(*s_column->field(1)); + for (int64_t i = 0; i < s_column->length(); ++i) { + EXPECT_EQ(sibling.GetString(i), NestedSiblingValue(static_cast(i))); + } + } + + // Opens the written file and returns the reader plus its imported file schema. + void OpenFile(std::unique_ptr* file_reader, + std::shared_ptr* file_schema) { + ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); + auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); + auto in_stream = + std::make_unique(std::move(input_stream), length, arrow_pool_); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(auto parquet_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, + /*batch_size=*/1024, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + *file_reader = std::move(parquet_reader); + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema, + (*file_reader)->GetFileSchema()); + auto imported = arrow::ImportSchema(c_file_schema.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + *file_schema = imported.ValueOrDie(); + } + + // Builds a variant-access projection field for a variant column via the public builder. + std::shared_ptr BuildAccessField( + const std::vector, std::string>>& accesses, + const std::string& field_name = "v") { + VariantAccessBuilder builder; + for (const auto& [type, path] : accesses) { + auto c_target = std::make_unique(); + EXPECT_TRUE(arrow::ExportField(arrow::Field("t", type), c_target.get()).ok()); + EXPECT_OK(builder.AddField(c_target.get(), path, /*fail_on_error=*/false)); + } + auto c_field = builder.Build(field_name); + EXPECT_TRUE(c_field.ok()) << c_field.status().ToString(); + auto imported = arrow::ImportField(c_field.value().get()); + EXPECT_TRUE(imported.ok()) << imported.status().ToString(); + return imported.ValueOrDie(); + } + + // Reads the whole file through the shredding reader with the given read schema and returns + // the second (variant) column. + void ReadColumn(const std::shared_ptr& read_schema, + std::shared_ptr* column) { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + auto shredding_reader = + std::make_unique(std::move(file_reader), std::move(plans), pool_); + auto c_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + ASSERT_OK(shredding_reader->SetReadSchema(c_read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, shredding_reader->NextBatchWithBitmap()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch_with_bitmap)); + auto& [read_batch, bitmap] = batch_with_bitmap; + auto imported = arrow::ImportArray(read_batch.first.get(), read_batch.second.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + auto result_struct = std::static_pointer_cast(imported.ValueOrDie()); + *column = result_struct->field(1); + shredding_reader->Close(); + // The assembled arrays borrow the reader's memory pool; keep the reader alive until the + // fixture is torn down (fixture members outlive test-body locals). + live_readers_.push_back(std::move(shredding_reader)); + } + + // `ReadColumn` for the cases whose second column is a struct. + void ReadVariantColumn(const std::shared_ptr& read_schema, + std::shared_ptr* v_column) { + std::shared_ptr column; + ReadColumn(read_schema, &column); + ASSERT_EQ(column->type_id(), arrow::Type::STRUCT); + *v_column = std::static_pointer_cast(column); + } + + protected: + std::unique_ptr dir_; + std::shared_ptr fs_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; + std::string file_path_; + std::shared_ptr paimon_schema_; + std::shared_ptr nested_schema_; + std::shared_ptr nested_variant_field_; + std::shared_ptr nested_sibling_field_; + std::shared_ptr list_schema_; + std::shared_ptr list_element_field_; + std::shared_ptr map_schema_; + std::shared_ptr map_item_field_; + std::shared_ptr list_struct_schema_; + std::shared_ptr list_struct_variant_field_; + std::vector> live_readers_; +}; + +namespace { + +constexpr const char* kAgeCityShreddingSchema = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ + {"id": 1, "name": "age", "type": "INT"}, + {"id": 2, "name": "city", "type": "STRING"} + ] + } + } ] +})"; + +} // namespace + +TEST_F(VariantParquetTest, PhysicalLayoutMatchesJava) { + auto array = BuildArray({R"({"a": 1, "b": "hello"})", nullptr, "[1,2,3]"}); + WriteFile(array); + + // The on-disk layout must match the Java ParquetSchemaConverter: an (unannotated) group + // with two REQUIRED BINARY fields `value` (id 0) and `metadata` (id 1). The raw parquet + // reader is required because these parquet-level properties (repetition, physical types, + // field ids, the absence of a logical-type annotation) are not visible in the Arrow schema + // surfaced by the paimon reader. + auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get()); + ASSERT_TRUE(file.ok()); + std::unique_ptr<::parquet::arrow::FileReader> reader; + auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader); + ASSERT_TRUE(status.ok()) << status.ToString(); + const ::parquet::SchemaDescriptor* schema = reader->parquet_reader()->metadata()->schema(); + ASSERT_EQ(schema->num_columns(), 3); + const auto* root = schema->group_node(); + ASSERT_EQ(root->field_count(), 2); + const auto& variant_group_node = root->field(1); + ASSERT_TRUE(variant_group_node->is_group()); + ASSERT_EQ(variant_group_node->name(), "v"); + ASSERT_EQ(variant_group_node->field_id(), 2); + ASSERT_EQ(variant_group_node->logical_type()->type(), ::parquet::LogicalType::Type::NONE); + const auto* variant_group = + static_cast(variant_group_node.get()); + ASSERT_EQ(variant_group->field_count(), 2); + const auto& value_node = variant_group->field(0); + ASSERT_EQ(value_node->name(), "value"); + ASSERT_TRUE(value_node->is_primitive()); + ASSERT_TRUE(value_node->is_required()); + ASSERT_EQ(value_node->field_id(), 0); + ASSERT_EQ( + static_cast(value_node.get())->physical_type(), + ::parquet::Type::BYTE_ARRAY); + const auto& metadata_node = variant_group->field(1); + ASSERT_EQ(metadata_node->name(), "metadata"); + ASSERT_TRUE(metadata_node->is_primitive()); + ASSERT_TRUE(metadata_node->is_required()); + ASSERT_EQ(metadata_node->field_id(), 1); +} + +TEST_F(VariantParquetTest, WriteAndReadRoundTrip) { + std::vector jsons = { + R"({"a": 1, "b": "hello"})", + nullptr, + "[1,2,3]", + "{\"nested\": {\"x\": [true, null, 1.5]}, \"s\": \"中文\"}", + "12345678901234", + "100.99", + }; + auto array = BuildArray(jsons); + WriteFile(array); + + { + // Sanity-check the raw file through the plain parquet-arrow reader: the struct child + // arrays must align with the logical rows. + auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get()); + ASSERT_TRUE(file.ok()); + std::unique_ptr<::parquet::arrow::FileReader> raw_reader; + ASSERT_TRUE( + ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &raw_reader).ok()); + std::shared_ptr table; + ASSERT_TRUE(raw_reader->ReadTable(&table).ok()); + auto raw_variant = std::static_pointer_cast(table->column(1)->chunk(0)); + auto raw_value = std::static_pointer_cast(raw_variant->field(0)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("raw row " + std::to_string(i)); + if (jsons[i] != nullptr) { + ASSERT_FALSE(raw_variant->IsNull(i)); + ASSERT_GT(raw_value->GetView(i).size(), 0); + } else { + ASSERT_TRUE(raw_variant->IsNull(i)); + } + } + } + + ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); + auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); + auto in_stream = + std::make_unique(std::move(input_stream), length, arrow_pool_); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, + /*batch_size=*/1024, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*paimon_schema_, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto result_chunked, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + batch_reader->Close(); + ASSERT_EQ(result_chunked->length(), static_cast(jsons.size())); + ASSERT_EQ(result_chunked->num_chunks(), 1); + auto result_struct = std::static_pointer_cast(result_chunked->chunk(0)); + + auto variant_column = std::static_pointer_cast(result_struct->field(1)); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + ASSERT_EQ(variant_column->field(0)->length(), variant_column->length()); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + auto value_view = value_column->GetView(i); + auto metadata_view = metadata_column->GetView(i); + SCOPED_TRACE("value size " + std::to_string(value_view.size()) + ", metadata size " + + std::to_string(metadata_view.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_view, metadata_view, pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, ShreddedWriteAndReadRoundTrip) { + std::vector jsons = { + R"({"age": 35, "city": "Hangzhou"})", + nullptr, + R"({"age": "not a number", "extra": [1, 2]})", + "[\"top level array\"]", + }; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + auto file_variant_field = file_schema->GetFieldByName("v"); + ASSERT_NE(file_variant_field, nullptr); + ASSERT_TRUE(VariantShreddingUtils::IsShreddedFileType(file_variant_field->type())) + << file_variant_field->type()->ToString(); + file_reader->Close(); + } + + // Reading the column as a plain VARIANT reassembles every physical shape back to the + // original logical value. + std::shared_ptr variant_column; + ReadVariantColumn(paimon_schema_, &variant_column); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, UntypedPhysicalVariantWriteAndReadRoundTrip) { + std::vector jsons = { + R"({"a": 1, "b": "hello"})", + nullptr, + "[1,2,3]", + }; + WriteShreddedFile(jsons, arrow::null()); + + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + auto file_variant_field = file_schema->GetFieldByName("v"); + ASSERT_NE(file_variant_field, nullptr); + const auto& physical_type = + static_cast(*file_variant_field->type()); + ASSERT_EQ(physical_type.num_fields(), 2); + ASSERT_EQ(physical_type.field(0)->name(), VariantDefs::kMetadataFieldName); + ASSERT_EQ(physical_type.field(1)->name(), VariantDefs::kValueFieldName); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(file_variant_field->type())); + ASSERT_TRUE( + VariantShreddingUtils::IsUntypedPhysicalVariantType(file_variant_field->type())); + file_reader->Close(); + } + + std::shared_ptr variant_column; + ReadVariantColumn(paimon_schema_, &variant_column); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, AdaptiveInferenceUntypedPhysicalWriteAndReadRoundTrip) { + std::vector jsons = { + R"({"a": 1})", + "[1,2,3]", + nullptr, + }; + auto logical = BuildArray(jsons); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({ + {Options::MANIFEST_FORMAT, "parquet"}, + {Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"}, + {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"}, + })); + auto factory = VariantShreddingWritePlanFactory::Create(options, paimon_schema_, pool_); + std::vector> samples = {logical}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + auto physical_variant = converter->GetPhysicalSchema()->GetFieldByName("v"); + ASSERT_NE(physical_variant, nullptr); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(physical_variant->type())); + ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(physical_variant->type())); + + auto c_logical = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*logical, c_logical.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_logical.get())); + WriteFile(converter->GetPhysicalSchema(), c_physical.get()); + ASSERT_OK(factory->OnFileCompleted(converter)); + + std::shared_ptr variant_column; + ReadVariantColumn(paimon_schema_, &variant_column); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, VariantAccessReadMixedTypedAndBinary) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + auto access_field = BuildAccessField( + {{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}, {arrow::utf8(), "$.missing"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + // The plan prunes `typed_value` to the requested keys and keeps `value` because `$.other` + // and `$.missing` are not shredded. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + const auto& physical_type = + static_cast(*plans.at("v")->PhysicalField()->type()); + ASSERT_NE(physical_type.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr); + ASSERT_NE(physical_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_EQ(typed_struct.num_fields(), 1); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + file_reader->Close(); + } + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(v_column->length(), 3); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + const auto& missing = static_cast(*v_column->field(2)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + ASSERT_TRUE(missing.IsNull(0)); + ASSERT_TRUE(missing.IsNull(1)); +} + +TEST_F(VariantParquetTest, VariantAccessReadTypedOnlyPrunesValue) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})"}; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + auto access_field = BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.city"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + // All requested keys are shredded: neither `value` nor the unrequested typed keys are read. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + const auto& physical_type = + static_cast(*plans.at("v")->PhysicalField()->type()); + ASSERT_EQ(physical_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + ASSERT_EQ(typed_value->type()->num_fields(), 2); + file_reader->Close(); + } + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& age = static_cast(*v_column->field(0)); + const auto& city = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_EQ(city.GetString(0), "Chicago"); + // Row 1 has no "city" key: the shredded field is missing, which reads as null. + ASSERT_TRUE(city.IsNull(1)); +} + +TEST_F(VariantParquetTest, VariantAccessReadUnshreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteFile(BuildArray(jsons)); + + auto access_field = BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(v_column->length(), 3); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); +} + +TEST_F(VariantParquetTest, VariantAccessReadSemicolonKey) { + // Object keys may contain the description delimiter; the description parser anchors on the + // trailing failOnError/timeZoneId tokens instead of splitting on every delimiter. + std::vector jsons = {R"({"a;b": 7})"}; + WriteFile(BuildArray(jsons)); + auto access_field = BuildAccessField({{arrow::int64(), "$['a;b']"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(static_cast(*v_column->field(0)).Value(0), 7); +} + +TEST_F(VariantParquetTest, VariantAccessReadVariantTarget) { + std::vector jsons = {R"({"user": {"name": "Paimon", "age": 1}})", + R"({"user": "flat"})"}; + WriteFile(BuildArray(jsons)); + + // A variant-marked target re-encodes the extracted sub-variant instead of casting it to a + // plain struct; the marker on the target field must survive AddField. + VariantAccessBuilder builder; + ASSERT_OK_AND_ASSIGN(auto variant_target, Variant::ArrowField("t")); + ASSERT_OK(builder.AddField(variant_target.get(), "$.user")); + ASSERT_OK_AND_ASSIGN(auto c_access_field, builder.Build("v")); + auto imported = arrow::ImportField(c_access_field.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + auto read_schema = arrow::schema({paimon_schema_->field(0), imported.ValueOrDie()}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& user = static_cast(*v_column->field(0)); + const auto& value_column = static_cast(*user.field(0)); + const auto& metadata_column = static_cast(*user.field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row0, + GenericVariant::Create(value_column.GetView(0), metadata_column.GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row0_json, row0->ToJson()); + ASSERT_EQ(row0_json, R"({"age":1,"name":"Paimon"})"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row1, + GenericVariant::Create(value_column.GetView(1), metadata_column.GetView(1), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row1_json, row1->ToJson()); + ASSERT_EQ(row1_json, R"("flat")"); +} + +TEST_F(VariantParquetTest, VariantAccessReadNestedPath) { + const char* nested_shredding_schema = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ { + "id": 1, + "name": "address", + "type": { + "type": "ROW", + "fields": [ {"id": 2, "name": "city", "type": "STRING"} ] + } + } ] + } + } ] + })"; + std::vector jsons = {R"({"address": {"city": "Hangzhou"}})", + R"({"address": "oops"})", R"({"address": {"zip": 12345}})"}; + WriteShreddedFile(jsons, nested_shredding_schema); + + auto access_field = BuildAccessField({{arrow::utf8(), "$.address.city"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& city = static_cast(*v_column->field(0)); + ASSERT_EQ(city.GetString(0), "Hangzhou"); + // Row 1's address is not an object; row 2's address has no "city" key. + ASSERT_TRUE(city.IsNull(1)); + ASSERT_TRUE(city.IsNull(2)); +} + +TEST_F(VariantParquetTest, NestedVariantPlainReadReassembles) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", nullptr}; + WriteShreddedNestedFile(jsons, arrow::struct_({arrow::field("age", arrow::int32()), + arrow::field("city", arrow::utf8())})); + + // Read as a plain nested VARIANT: the shredded sub-columns are reassembled back into + // `struct`. + std::shared_ptr s_column; + ReadVariantColumn(nested_schema_, &s_column); + const auto& nv = static_cast(*s_column->field(0)); + ASSERT_TRUE(nv.type()->Equals(*nested_variant_field_->type())); + const auto& value_column = static_cast(*nv.field(0)); + const auto& metadata_column = static_cast(*nv.field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row0, + GenericVariant::Create(value_column.GetView(0), metadata_column.GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row0_json, row0->ToJson()); + ASSERT_EQ(row0_json, R"({"age":35,"city":"Chicago"})"); + ASSERT_TRUE(nv.IsNull(1)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, NestedVariantAccessReadShreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteShreddedNestedFile(jsons, arrow::struct_({arrow::field("age", arrow::int32()), + arrow::field("city", arrow::utf8())})); + + auto read_schema = + BuildNestedAccessReadSchema({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + + // A nested projection prunes the scan the same way a top-level one does: `typed_value` is + // narrowed to `age`, and `value` is kept because `$.other` is not shredded. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + const auto& physical_struct = + static_cast(*plans.at("s")->PhysicalField()->type()); + auto physical_nv = physical_struct.GetFieldByName("nv"); + ASSERT_NE(physical_nv, nullptr); + const auto& physical_nv_type = static_cast(*physical_nv->type()); + ASSERT_NE(physical_nv_type.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr); + ASSERT_NE(physical_nv_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_nv_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_EQ(typed_struct.num_fields(), 1); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + file_reader->Close(); + } + + std::shared_ptr s_column; + ReadVariantColumn(read_schema, &s_column); + ASSERT_EQ(s_column->length(), 3); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + // The extracted values must match the requested access struct, not `struct`. + ASSERT_EQ(nv.num_fields(), 2); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + // Row 2's nested variant is null, so the whole projection is null. + ASSERT_TRUE(nv.IsNull(2)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, NestedVariantAccessReadUnshreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteNestedFile(jsons); + + // The nested column is stored unshredded: the paths are extracted from the `value` binary, + // which still requires a read plan (the projection type must never reach the format reader). + auto read_schema = + BuildNestedAccessReadSchema({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + std::shared_ptr s_column; + ReadVariantColumn(read_schema, &s_column); + ASSERT_EQ(s_column->length(), 3); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + ASSERT_TRUE(nv.IsNull(2)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, ListVariantPlainReadNeedsNoPlan) { + // A variant inside a repeated group is never shredded, so a plain read of it still needs no + // plan at all: the logical type is exactly what the file stores. + WriteListFile({{R"({"x": 1})", R"({"x": 2})"}, {R"({"x": 3})"}}); + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + list_schema_, file_schema, pool_)); + ASSERT_TRUE(plans.empty()); + file_reader->Close(); +} + +TEST_F(VariantParquetTest, ListVariantAccessRead) { + // A variant-access projection inside an ARRAY extracts the paths per element (as in Java's + // testReadNestedVariantInArray). The empty row and the null element cover the offsets and + // the element validity being carried over by the reassembly. + WriteListFile( + {{R"({"x": 1, "y": 2})", R"({"x": 3, "y": 4})"}, {}, {R"({"x": 5, "y": 6})", nullptr}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "element"); + auto read_schema = arrow::schema( + {list_schema_->field(0), list_schema_->field(1)->WithType(arrow::list(access_field))}); + + // The parquet reader rejects partial projection inside a repeated group, so the whole file + // subtree is pushed down and only reassembled back. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + ASSERT_TRUE(plans.at("arr")->PhysicalField()->type()->Equals( + *file_schema->GetFieldByName("arr")->type())); + file_reader->Close(); + } + + std::shared_ptr arr_column; + ReadColumn(read_schema, &arr_column); + ASSERT_NE(arr_column, nullptr); + const auto& list = static_cast(*arr_column); + ASSERT_EQ(list.length(), 3); + ASSERT_EQ(list.value_length(0), 2); + ASSERT_EQ(list.value_length(1), 0); + ASSERT_EQ(list.value_length(2), 2); + const auto& elements = static_cast(*list.values()); + const auto& x = static_cast(*elements.field(0)); + ASSERT_EQ(x.Value(list.value_offset(0)), 1); + ASSERT_EQ(x.Value(list.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(list.value_offset(2)), 5); + // The null element's whole projection is null. + ASSERT_TRUE(elements.IsNull(list.value_offset(2) + 1)); +} + +TEST_F(VariantParquetTest, ListOfStructVariantAccessRead) { + // A variant one struct level below the ARRAY: the plan descends list -> struct -> variant + // and must leave the struct's plain sibling untouched. + WriteListStructFile({{R"({"x": 1, "y": 2})", R"({"x": 3})"}, {R"({"x": 5})"}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "v"); + auto element_field = + arrow::field("element", arrow::struct_({access_field, nested_sibling_field_})); + auto read_schema = + arrow::schema({list_struct_schema_->field(0), + list_struct_schema_->field(1)->WithType(arrow::list(element_field))}); + + std::shared_ptr arr_column; + ReadColumn(read_schema, &arr_column); + ASSERT_NE(arr_column, nullptr); + const auto& list = static_cast(*arr_column); + ASSERT_EQ(list.length(), 2); + const auto& elements = static_cast(*list.values()); + // Each element is `struct, t: STRING>`, so the extracted path sits one + // struct level below the element. + const auto& access = static_cast(*elements.field(0)); + const auto& x = static_cast(*access.field(0)); + const auto& sibling = static_cast(*elements.field(1)); + ASSERT_EQ(x.Value(0), 1); + ASSERT_EQ(x.Value(1), 3); + ASSERT_EQ(x.Value(2), 5); + for (int64_t i = 0; i < 3; ++i) { + EXPECT_EQ(sibling.GetString(i), NestedSiblingValue(static_cast(i))); + } +} + +TEST_F(VariantParquetTest, ListPartialProjectionNeedsNoPlan) { + // Projecting a subset of a struct inside a repeated group is unsupported, so no plan is + // built: the read fails in the reader instead of assembling a mistyped column. + WriteListStructFile({{R"({"x": 1})"}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "v"); + auto element_field = arrow::field("element", arrow::struct_({access_field})); + auto read_schema = + arrow::schema({list_struct_schema_->field(0), + list_struct_schema_->field(1)->WithType(arrow::list(element_field))}); + + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_TRUE(plans.empty()); + file_reader->Close(); +} + +TEST_F(VariantParquetTest, MapVariantAccessRead) { + // The same for a variant value inside a MAP: the plan descends through the map entries + // struct and rewrites only the item child. + WriteMapFile({{"a", "b"}, {"c", "d"}}, + {{R"({"x": 1, "y": 2})", R"({"x": 3})"}, {R"({"x": 5, "y": 6})", nullptr}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "value"); + auto read_schema = + arrow::schema({map_schema_->field(0), + map_schema_->field(1)->WithType(arrow::map(arrow::utf8(), access_field))}); + + std::shared_ptr m_column; + ReadColumn(read_schema, &m_column); + ASSERT_NE(m_column, nullptr); + const auto& map = static_cast(*m_column); + ASSERT_EQ(map.length(), 2); + ASSERT_EQ(map.value_length(0), 2); + ASSERT_EQ(map.value_length(1), 2); + // The keys must survive untouched next to the rewritten values. + const auto& keys = static_cast(*map.keys()); + ASSERT_EQ(keys.GetString(map.value_offset(0)), "a"); + ASSERT_EQ(keys.GetString(map.value_offset(0) + 1), "b"); + ASSERT_EQ(keys.GetString(map.value_offset(1)), "c"); + ASSERT_EQ(keys.GetString(map.value_offset(1) + 1), "d"); + const auto& items = static_cast(*map.items()); + const auto& x = static_cast(*items.field(0)); + ASSERT_EQ(x.Value(map.value_offset(0)), 1); + ASSERT_EQ(x.Value(map.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(map.value_offset(1)), 5); + // A null map value projects to a null row. + ASSERT_TRUE(items.IsNull(map.value_offset(1) + 1)); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/fs/jindo/jindo_file_status.h b/src/paimon/fs/jindo/jindo_file_status.h index 5ac14cfd..bc291eb0 100644 --- a/src/paimon/fs/jindo/jindo_file_status.h +++ b/src/paimon/fs/jindo/jindo_file_status.h @@ -49,7 +49,7 @@ class JindoFileStatus : public FileStatus { return file_info_.getPath(); } - uint64_t GetLen() const override { + int64_t GetLen() const override { return file_info_.getLength(); } diff --git a/src/paimon/fs/jindo/jindo_file_system.cpp b/src/paimon/fs/jindo/jindo_file_system.cpp index afd5a383..0bfb7c8c 100644 --- a/src/paimon/fs/jindo/jindo_file_system.cpp +++ b/src/paimon/fs/jindo/jindo_file_system.cpp @@ -18,7 +18,9 @@ #include "paimon/fs/jindo/jindo_file_system.h" +#include #include +#include #include #include "JdoFileInfo.hpp" // NOLINT(build/include_subdir) @@ -27,6 +29,8 @@ #include "JdoStatus.hpp" // NOLINT(build/include_subdir) #include "fmt/format.h" #include "jdo_error.h" // NOLINT(build/include_subdir) +#include "paimon/common/utils/math.h" +#include "paimon/common/utils/path_util.h" #include "paimon/fs/jindo/jindo_file_status.h" #include "paimon/fs/jindo/jindo_utils.h" @@ -53,6 +57,29 @@ class JindoFileSystemImpl { std::unique_ptr fs_; }; +namespace { + +class AsyncReadState { + public: + explicit AsyncReadState(std::function&& callback) + : callback_(std::move(callback)) {} + + void Complete(JdoStatus status) { + if (completed_.exchange(true)) { + return; + } + callback_(status.ok() ? Status::OK() : Status::IOError(status.errMsg())); + } + + std::string_view result; + + private: + std::atomic completed_{false}; + std::function callback_; +}; + +} // namespace + JindoFileSystem::JindoFileSystem(std::unique_ptr&& fs) : impl_(std::make_shared(std::move(fs))) {} @@ -69,6 +96,18 @@ Result> JindoFileSystem::Create(const std::string& return Status::Invalid( fmt::format("do not allow overwrite, but the file {} already exists", path)); } + const std::string parent_path = PathUtil::GetParentDirPath(path); + if (!parent_path.empty()) { + PAIMON_ASSIGN_OR_RAISE(Path parent, PathUtil::ToPath(parent_path)); + // Do not issue mkdir for scheme-only or authority-only URI parents. + if (!parent.path.empty()) { + PAIMON_RETURN_NOT_OK(Mkdirs(parent_path)); + } + } + return OpenWriter(path); +} + +Result> JindoFileSystem::OpenWriter(const std::string& path) const { std::unique_ptr writer; PAIMON_RETURN_NOT_OK_FROM_JINDO(impl_->GetFileSystem()->openWriter(path, &writer)); return std::make_unique(impl_, std::move(writer)); @@ -192,7 +231,7 @@ Status JindoInputStream::Seek(int64_t offset, SeekOrigin origin) { PAIMON_ASSIGN_OR_RAISE(int64_t pos, GetPos()); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->seek(offset + pos)); } else if (origin == FS_SEEK_END) { - PAIMON_ASSIGN_OR_RAISE(uint64_t len, Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t len, Length()); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->seek(len + offset)); } else { return Status::Invalid("unsupported seek origin"); @@ -203,33 +242,55 @@ Status JindoInputStream::Seek(int64_t offset, SeekOrigin origin) { Result JindoInputStream::GetPos() const { int64_t pos = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->tell(pos)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(pos, "jindo input position")); return pos; } -Result JindoInputStream::Length() const { +Result JindoInputStream::Length() const { int64_t len = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->getFileLength(len)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(len, "jindo input length")); return len; } -Result JindoInputStream::Read(char* buffer, uint32_t size) { - PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->read(size, &result_, buffer)); - return result_.length(); +Result JindoInputStream::Read(char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + std::string_view result; + PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->read(size, &result, buffer)); + return result.length(); } -Result JindoInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { - PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->pread(offset, size, &result_, buffer)); - return result_.length(); +Result JindoInputStream::Read(char* buffer, int64_t size, int64_t offset) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); + std::string_view result; + PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->pread(offset, size, &result, buffer)); + return result.length(); } -void JindoInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void JindoInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { - auto outer_callback = [=](JdoStatus status) { - callback(status.ok() ? Status::OK() : Status::IOError(status.errMsg())); - }; - auto task = reader_->preadAsync(offset, size, &result_, buffer, outer_callback); + Status validate_status = ValidateValueNonNegative(size, "read length"); + if (!validate_status.ok()) { + callback(validate_status); + return; + } + validate_status = ValidateValueNonNegative(offset, "read offset"); + if (!validate_status.ok()) { + callback(validate_status); + return; + } + std::shared_ptr state = std::make_shared(std::move(callback)); + auto task = reader_->preadAsync(offset, size, &state->result, buffer, + [state](JdoStatus status) { state->Complete(status); }); assert(task); - auto status = task->perform(); + + auto perform_status = task->perform(); + if (!perform_status.ok()) { + state->Complete(perform_status); + [[maybe_unused]] auto status = task->cancel(); + return; + } } Status JindoInputStream::Close() { @@ -250,10 +311,12 @@ JindoOutputStream::JindoOutputStream(const std::shared_ptr& Result JindoOutputStream::GetPos() const { int64_t pos = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(writer_->tell(pos)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(pos, "jindo output position")); return pos; } -Result JindoOutputStream::Write(const char* buffer, uint32_t size) { +Result JindoOutputStream::Write(const char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "write length")); std::string_view data(buffer, size); PAIMON_RETURN_NOT_OK_FROM_JINDO(writer_->write(data)); return size; diff --git a/src/paimon/fs/jindo/jindo_file_system.h b/src/paimon/fs/jindo/jindo_file_system.h index ce5d1edf..0ba8990b 100644 --- a/src/paimon/fs/jindo/jindo_file_system.h +++ b/src/paimon/fs/jindo/jindo_file_system.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "JdoFileSystem.hpp" // NOLINT(build/include_subdir) @@ -59,6 +58,9 @@ class JindoFileSystem : public FileSystem { Result Exists(const std::string& path) const override; + protected: + virtual Result> OpenWriter(const std::string& path) const; + private: std::shared_ptr impl_; }; @@ -69,20 +71,19 @@ class JindoInputStream : public InputStream { std::unique_ptr&& reader); Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + Result Read(char* buffer, int64_t size) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; Status Close() override; Result GetUri() const override; - Result Length() const override; + Result Length() const override; private: // The lifecycle of the fs used to create the Jindo Reader must be longer than the lifecycle of // the Jindo Reader. std::shared_ptr fs_; std::unique_ptr reader_; - std::string_view result_; }; class JindoOutputStream : public OutputStream { @@ -91,7 +92,7 @@ class JindoOutputStream : public OutputStream { std::unique_ptr&& writer); Result GetPos() const override; - Result Write(const char* buffer, uint32_t size) override; + Result Write(const char* buffer, int64_t size) override; Status Flush() override; Status Close() override; Result GetUri() const override; diff --git a/src/paimon/fs/jindo/jindo_file_system_test.cpp b/src/paimon/fs/jindo/jindo_file_system_test.cpp index 39b8b577..efaf2a72 100644 --- a/src/paimon/fs/jindo/jindo_file_system_test.cpp +++ b/src/paimon/fs/jindo/jindo_file_system_test.cpp @@ -16,10 +16,19 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include + #include "gtest/gtest.h" #include "paimon/fs/jindo/jindo_file_system_factory.h" #include "paimon/testing/utils/testharness.h" + namespace paimon::jindo::test { + // This test shows inconsistent behavior with the local file system in some abnormal scenarios. class JindoFileSystemTest : public ::testing::Test { public: @@ -52,7 +61,7 @@ TEST_F(JindoFileSystemTest, TestLifeCycle) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, tmp_fs->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -79,7 +88,7 @@ TEST_F(JindoFileSystemTest, TestSeek) { std::string file_path = test_dir_ + "file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -109,7 +118,7 @@ TEST_F(JindoFileSystemTest, TestSeek) { // read from cur pos std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("ijk", read_content); @@ -126,4 +135,73 @@ TEST_F(JindoFileSystemTest, TestSeek) { ASSERT_OK(in_stream->Close()); } +TEST(JindoFileSystemPaginationTest, TestListDirAcrossOssPageBoundary) { + constexpr int32_t kFileCount = 1234; + const std::string test_dir = "oss://paimon-unittest/test_data/jindo_listdir_truncated_1234/"; + std::map options = paimon::test::GetJindoTestOptions(); + + auto fs_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs, fs_factory->Create(test_dir, options)); + + std::vector> file_statuses; + ASSERT_OK(fs->ListDir(test_dir, &file_statuses)); + ASSERT_EQ(file_statuses.size(), kFileCount); + + std::unordered_set actual_paths; + for (const std::unique_ptr& file_status : file_statuses) { + ASSERT_TRUE(actual_paths.insert(file_status->GetPath()).second) + << "duplicate path: " << file_status->GetPath(); + } + for (int32_t i = 0; i < kFileCount; ++i) { + std::string index = std::to_string(i); + index.insert(/*pos=*/0, /*count=*/4 - index.size(), /*ch=*/'0'); + ASSERT_NE(actual_paths.find(test_dir + "file-" + index + ".txt"), actual_paths.end()); + } +} + +TEST(JindoFileSystemAsyncReadTest, TestConcurrentReadAsyncAndReadFromOss) { + constexpr int32_t kConcurrentReads = 64; + constexpr int64_t kAsyncReadSize = 7; + const std::string file_path = "oss://paimon-unittest/test_data/jindo_read_async_128mb.bin"; + std::map options = paimon::test::GetJindoTestOptions(); + auto fs_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs, fs_factory->Create(file_path, options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr input_stream, fs->Open(file_path)); + + std::vector> async_buffers(kConcurrentReads, + std::vector(kAsyncReadSize)); + std::vector> promises(kConcurrentReads); + std::vector> futures; + futures.reserve(kConcurrentReads); + for (std::promise& promise : promises) { + futures.push_back(promise.get_future()); + } + + std::vector sync_read_statuses; + std::vector sync_read_sizes; + sync_read_statuses.reserve(kConcurrentReads); + sync_read_sizes.reserve(kConcurrentReads); + for (int32_t i = 0; i < kConcurrentReads; ++i) { + input_stream->ReadAsync( + async_buffers[i].data(), async_buffers[i].size(), /*offset=*/0, + [&promises, i](Status status) { promises[i].set_value(std::move(status)); }); + + char sync_buffer = 0; + Result sync_read_result = + input_stream->Read(&sync_buffer, /*size=*/1, /*offset=*/i); + sync_read_statuses.push_back(sync_read_result.status()); + sync_read_sizes.push_back(sync_read_result.ok() ? sync_read_result.value() : -1); + } + + for (int32_t i = 0; i < kConcurrentReads; ++i) { + ASSERT_EQ(futures[i].wait_for(std::chrono::seconds(60)), std::future_status::ready) + << "async read=" << i; + ASSERT_OK(futures[i].get()); + } + for (int32_t i = 0; i < kConcurrentReads; ++i) { + ASSERT_OK(sync_read_statuses[i]) << "sync read=" << i; + ASSERT_EQ(sync_read_sizes[i], 1) << "sync read=" << i; + } +} + } // namespace paimon::jindo::test diff --git a/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp b/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp new file mode 100644 index 00000000..11a4c76e --- /dev/null +++ b/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/fs/jindo/jindo_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class TestOutputStream : public OutputStream { + public: + Result GetPos() const override { + return 0; + } + + Result Write(const char*, int64_t size) override { + return size; + } + + Status Flush() override { + return Status::OK(); + } + + Status Close() override { + return Status::OK(); + } + + Result GetUri() const override { + return std::string(); + } +}; + +class TestJindoFileSystem : public jindo::JindoFileSystem { + public: + TestJindoFileSystem() : JindoFileSystem(std::make_unique()) {} + + Result Exists(const std::string&) const override { + return false; + } + + Status Mkdirs(const std::string& path) const override { + parent_path_ = path; + calls_.push_back("mkdirs"); + return mkdirs_status_; + } + + void SetMkdirsStatus(Status status) { + mkdirs_status_ = std::move(status); + } + + const std::string& GetParentPath() const { + return parent_path_; + } + + bool IsWriterOpened() const { + return writer_opened_; + } + + const std::vector& GetCalls() const { + return calls_; + } + + protected: + Result> OpenWriter(const std::string&) const override { + calls_.push_back("open_writer"); + writer_opened_ = true; + std::unique_ptr output = std::make_unique(); + return output; + } + + private: + mutable std::string parent_path_; + mutable bool writer_opened_ = false; + mutable std::vector calls_; + Status mkdirs_status_ = Status::OK(); +}; + +TEST(JindoFileSystemUnitTest, CreateMakesParentDirectoryBeforeOpeningWriter) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/table/bucket-24/data.orc"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + fs.Create(path, /*overwrite=*/false)); + ASSERT_TRUE(output); + ASSERT_EQ(fs.GetParentPath(), "oss://bucket/table/bucket-24"); + ASSERT_TRUE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 2); + ASSERT_EQ(fs.GetCalls()[0], "mkdirs"); + ASSERT_EQ(fs.GetCalls()[1], "open_writer"); +} + +TEST(JindoFileSystemUnitTest, CreateSkipsObjectStoreRootParent) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/data.orc"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + fs.Create(path, /*overwrite=*/false)); + ASSERT_TRUE(output); + ASSERT_TRUE(fs.GetParentPath().empty()); + ASSERT_TRUE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 1); + ASSERT_EQ(fs.GetCalls()[0], "open_writer"); +} + +TEST(JindoFileSystemUnitTest, CreateReturnsParentDirectoryFailure) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/table/bucket-24/data.orc"; + fs.SetMkdirsStatus(Status::IOError("failed to create parent directory")); + + ASSERT_NOK_WITH_MSG(fs.Create(path, /*overwrite=*/false), "failed to create parent directory"); + ASSERT_EQ(fs.GetParentPath(), "oss://bucket/table/bucket-24"); + ASSERT_FALSE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 1); + ASSERT_EQ(fs.GetCalls()[0], "mkdirs"); +} + +} // namespace paimon::test diff --git a/src/paimon/fs/jindo/jindo_utils.h b/src/paimon/fs/jindo/jindo_utils.h index 9dfcc9d4..ca3d3a69 100644 --- a/src/paimon/fs/jindo/jindo_utils.h +++ b/src/paimon/fs/jindo/jindo_utils.h @@ -19,15 +19,21 @@ #pragma once #include "JdoStatus.hpp" // NOLINT(build/include_subdir) +#include "jdo_error.h" // NOLINT(build/include_subdir) #include "paimon/status.h" namespace paimon::jindo { -#define PAIMON_RETURN_NOT_OK_FROM_JINDO(JINDO_STATUS) \ - do { \ - auto __s = (JINDO_STATUS); \ - if (PAIMON_UNLIKELY(!(__s).ok())) { \ - return Status::IOError(__s.errMsg()); \ - } \ +/// Maps a jindo file-not-found error to Status::NotExist so that callers can distinguish +/// a missing file from other IO errors, consistent with LocalFileSystem. +#define PAIMON_RETURN_NOT_OK_FROM_JINDO(JINDO_STATUS) \ + do { \ + auto __s = (JINDO_STATUS); \ + if (PAIMON_UNLIKELY(!(__s).ok())) { \ + if ((__s).getErrCode() == JDO_FILE_NOT_FOUND_ERROR) { \ + return Status::NotExist(__s.errMsg()); \ + } \ + return Status::IOError(__s.errMsg()); \ + } \ } while (false) } // namespace paimon::jindo diff --git a/src/paimon/fs/jindo/jindo_utils_test.cpp b/src/paimon/fs/jindo/jindo_utils_test.cpp new file mode 100644 index 00000000..635c081d --- /dev/null +++ b/src/paimon/fs/jindo/jindo_utils_test.cpp @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/fs/jindo/jindo_utils.h" + +#include "gtest/gtest.h" +#include "jdo_error.h" // NOLINT(build/include_subdir) +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::jindo::test { + +namespace { + +Status Convert(const JdoStatus& jdo_status) { + PAIMON_RETURN_NOT_OK_FROM_JINDO(jdo_status); + return Status::OK(); +} + +} // namespace + +TEST(JindoUtilsTest, TestMacroMapsStatusByErrorCode) { + ASSERT_OK(Convert(JdoStatus())); + + Status not_found = Convert(JdoStatus(JDO_FILE_NOT_FOUND_ERROR, "file not found")); + ASSERT_TRUE(not_found.IsNotExist()); + ASSERT_NOK_WITH_MSG(not_found, "file not found"); + + Status other = Convert(JdoStatus(JDO_FILE_NOT_FOUND_ERROR + 1, "some other error")); + ASSERT_TRUE(other.IsIOError()); + ASSERT_NOK_WITH_MSG(other, "some other error"); +} + +} // namespace paimon::jindo::test diff --git a/src/paimon/fs/local/local_file.cpp b/src/paimon/fs/local/local_file.cpp index 0291c59d..c603154c 100644 --- a/src/paimon/fs/local/local_file.cpp +++ b/src/paimon/fs/local/local_file.cpp @@ -28,6 +28,7 @@ #include "fmt/format.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/math.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/fs/local/local_file_status.h" @@ -171,7 +172,7 @@ Result> LocalFile::GetFileStatus() const { S_ISDIR(buf.st_mode)); } -Result LocalFile::Length() const { +Result LocalFile::Length() const { CHECK_HOOK(); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, GetFileStatus()); return file_status->GetLen(); @@ -197,18 +198,16 @@ const std::string& LocalFile::GetPath() const { return path_; } -Result LocalFile::Read(char* buffer, uint32_t length, uint64_t offset) { +Result LocalFile::Read(char* buffer, int64_t length, int64_t offset) { if (file_) { CHECK_HOOK(); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); int32_t fd = fileno(file_); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError(fmt::format( - "pread file '{}' fail, length overflow int32_t, ec: EC_BADARGS", path_)); - } - uint64_t off = 0; - int32_t ret = 0; + int64_t more = length; + int64_t off = 0; + int64_t ret = 0; while (more > 0) { ret = ::pread(fd, buffer + off, more, offset + off); if (ret == -1) { @@ -227,17 +226,14 @@ Result LocalFile::Read(char* buffer, uint32_t length, uint64_t offset) "read file '{}' fail, can not read file which is opened fail, ec: EBADF", path_)); } -Result LocalFile::Read(char* buffer, uint32_t length) { +Result LocalFile::Read(char* buffer, int64_t length) { if (file_) { CHECK_HOOK(); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError( - fmt::format("fileName '{}', length '{}', ec: EC_BADARGS", path_, length)); - } + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); - int32_t ret = 0; - uint64_t off = 0; + int64_t more = length; + int64_t ret = 0; + int64_t off = 0; while (more > 0) { ret = fread(buffer + off, 1, more, file_); if (ferror(file_) != 0) { @@ -257,23 +253,25 @@ Result LocalFile::Read(char* buffer, uint32_t length) { "read file '{}' fail, can not read file which is opened fail, ec: EBADF", path_)); } -Result LocalFile::Write(const char* buffer, uint32_t length) { +Result LocalFile::Write(const char* buffer, int64_t length) { if (file_) { CHECK_HOOK(); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError(fmt::format( - "write file '{}' fail, length overflow int32_t, ec: EC_BADARGS", path_)); - } + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "write length")); - int32_t ret = 0; - uint64_t off = 0; + int64_t more = length; + int64_t ret = 0; + int64_t off = 0; while (more > 0) { ret = fwrite(buffer + off, 1, more, file_); if (ferror(file_) != 0) { return Status::IOError(fmt::format("write file '{}' fail at off {}, ec: {}", path_, off, std::strerror(errno))); } + if (ret == 0) { + return Status::IOError( + fmt::format("write file '{}' fail at off {}, wrote zero bytes, ec: {}", path_, + off, std::strerror(errno))); + } more -= ret; off += ret; } diff --git a/src/paimon/fs/local/local_file.h b/src/paimon/fs/local/local_file.h index 655c4141..7ebfd13f 100644 --- a/src/paimon/fs/local/local_file.h +++ b/src/paimon/fs/local/local_file.h @@ -53,12 +53,12 @@ class LocalFile { std::unique_ptr GetParentFile() const; Result Mkdir() const; Result> GetFileStatus() const; - Result Length() const; + Result Length() const; Result LastModifiedTimeMs() const; Status OpenFile(bool is_read_file); - Result Read(char* buffer, uint32_t length); - Result Read(char* buffer, uint32_t length, uint64_t offset); - Result Write(const char* buffer, uint32_t length); + Result Read(char* buffer, int64_t length); + Result Read(char* buffer, int64_t length, int64_t offset); + Result Write(const char* buffer, int64_t length); Status Flush(); Status Close(); Status Seek(int64_t offset, int32_t seek_origin); diff --git a/src/paimon/fs/local/local_file_status.h b/src/paimon/fs/local/local_file_status.h index eb1af1fd..8987d0be 100644 --- a/src/paimon/fs/local/local_file_status.h +++ b/src/paimon/fs/local/local_file_status.h @@ -43,7 +43,7 @@ class LocalBasicFileStatus : public BasicFileStatus { class LocalFileStatus : public FileStatus { public: - LocalFileStatus(const std::string& path, uint64_t length, int64_t last_modification_time, + LocalFileStatus(const std::string& path, int64_t length, int64_t last_modification_time, bool is_dir) : path_(path), length_(length), @@ -54,7 +54,7 @@ class LocalFileStatus : public FileStatus { return path_; } - uint64_t GetLen() const override { + int64_t GetLen() const override { return length_; } @@ -68,7 +68,7 @@ class LocalFileStatus : public FileStatus { private: std::string path_; - uint64_t length_; + int64_t length_; int64_t last_modification_time_; bool is_dir_; }; diff --git a/src/paimon/fs/local/local_file_system.cpp b/src/paimon/fs/local/local_file_system.cpp index 258a39f4..917506be 100644 --- a/src/paimon/fs/local/local_file_system.cpp +++ b/src/paimon/fs/local/local_file_system.cpp @@ -256,37 +256,37 @@ Result LocalInputStream::GetPos() const { return file_->Tell(); } -Result LocalInputStream::Read(char* buffer, uint32_t size) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, file_->Read(buffer, size)); - if (read_length != static_cast(size)) { +Result LocalInputStream::Read(char* buffer, int64_t size) { + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, file_->Read(buffer, size)); + if (read_length != size) { return Status::IOError(fmt::format("file '{}' read size {} != expected {}", file_->GetPath(), read_length, size)); } return read_length; } -Result LocalInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, file_->Read(buffer, size, offset)); - if (read_length != static_cast(size)) { +Result LocalInputStream::Read(char* buffer, int64_t size, int64_t offset) { + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, file_->Read(buffer, size, offset)); + if (read_length != size) { return Status::IOError(fmt::format("file '{}' read size {} != expected {}", file_->GetPath(), read_length, size)); } return read_length; } -void LocalInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void LocalInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { - Result read_size = Read(buffer, size, offset); + Result read_size = Read(buffer, size, offset); Status status = Status::OK(); if (!read_size.ok()) { status = read_size.status(); } else { - assert(read_size.value() == static_cast(size)); + assert(read_size.value() == size); } callback(status); } -Result LocalInputStream::Length() const { +Result LocalInputStream::Length() const { return file_->Length(); } @@ -306,7 +306,7 @@ LocalOutputStream::LocalOutputStream(std::unique_ptr&& file) : file_( Result LocalOutputStream::GetPos() const { return file_->Tell(); } -Result LocalOutputStream::Write(const char* buffer, uint32_t size) { +Result LocalOutputStream::Write(const char* buffer, int64_t size) { return file_->Write(buffer, size); } Status LocalOutputStream::Flush() { diff --git a/src/paimon/fs/local/local_file_system.h b/src/paimon/fs/local/local_file_system.h index 3a4bc7ab..748f2463 100644 --- a/src/paimon/fs/local/local_file_system.h +++ b/src/paimon/fs/local/local_file_system.h @@ -69,16 +69,16 @@ class LocalInputStream : public InputStream { Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + Result Read(char* buffer, int64_t size) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; Status Close() override; Result GetUri() const override { return file_->GetPath(); } - Result Length() const override; + Result Length() const override; private: explicit LocalInputStream(std::unique_ptr&& file); @@ -91,7 +91,7 @@ class LocalOutputStream : public OutputStream { static Result> Create(std::unique_ptr file); Result GetPos() const override; - Result Write(const char* buffer, uint32_t size) override; + Result Write(const char* buffer, int64_t size) override; Status Flush() override; Status Close() override; Result GetUri() const override { diff --git a/src/paimon/fs/local/local_file_test.cpp b/src/paimon/fs/local/local_file_test.cpp index 4ad3b971..f22095c9 100644 --- a/src/paimon/fs/local/local_file_test.cpp +++ b/src/paimon/fs/local/local_file_test.cpp @@ -48,8 +48,8 @@ TEST(LocalFileTest, TestReadWriteEmptyContent) { ASSERT_OK(file->OpenFile(/*is_read_file=*/false)); const char* str = ""; - const int32_t str_size = 0; - ASSERT_OK_AND_ASSIGN(int32_t write_size, file->Write(str, str_size)); + constexpr int64_t str_size = 0; + ASSERT_OK_AND_ASSIGN(int64_t write_size, file->Write(str, str_size)); ASSERT_EQ(write_size, str_size); ASSERT_OK(file->Flush()); @@ -60,7 +60,7 @@ TEST(LocalFileTest, TestReadWriteEmptyContent) { ASSERT_OK_AND_ASSIGN(auto file2, LocalFile::Create(path)); ASSERT_OK(file2->OpenFile(/*is_read_file=*/true)); char buffer[10]; - ASSERT_OK_AND_ASSIGN(int32_t read_len, file2->Read(buffer, 10)); + ASSERT_OK_AND_ASSIGN(int64_t read_len, file2->Read(buffer, 10)); ASSERT_EQ(0, read_len); } @@ -85,8 +85,8 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK(file->OpenFile(/*is_read_file=*/false)); const char* str = "test_data"; - const int32_t str_size = 9; - ASSERT_OK_AND_ASSIGN(int32_t write_size, file->Write(str, str_size)); + constexpr int64_t str_size = 9; + ASSERT_OK_AND_ASSIGN(int64_t write_size, file->Write(str, str_size)); ASSERT_EQ(write_size, str_size); ASSERT_OK(file->Flush()); @@ -101,7 +101,7 @@ TEST(LocalFileTest, TestSimple) { std::vector file_list; ASSERT_NOK(file->List(&file_list)); - ASSERT_OK_AND_ASSIGN(size_t len, file->Length()); + ASSERT_OK_AND_ASSIGN(int64_t len, file->Length()); ASSERT_EQ(len, str_size); ASSERT_OK_AND_ASSIGN(auto file2, LocalFile::Create(path)); @@ -110,7 +110,7 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK(file2->OpenFile(true)); char str_read[str_size + 1]; { - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, 4)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, 4)); ASSERT_EQ(read_size, 4); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "test"), 0); @@ -119,13 +119,13 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK_AND_ASSIGN(int64_t pos, file2->Tell()); ASSERT_EQ(pos, 4); ASSERT_OK(file2->Seek(5, SEEK_SET)); - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, 4)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, 4)); ASSERT_EQ(read_size, 4); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "data"), 0); } { - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, str_size, 0)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, str_size, 0)); ASSERT_EQ(read_size, str_size); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "test_data"), 0); diff --git a/src/paimon/fs/s3/CMakeLists.txt b/src/paimon/fs/s3/CMakeLists.txt new file mode 100644 index 00000000..4c6a0d0e --- /dev/null +++ b/src/paimon/fs/s3/CMakeLists.txt @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(PAIMON_ENABLE_S3) + add_paimon_lib(paimon_s3_file_system + SOURCES + s3_file_system.cpp + s3_file_system_factory.cpp + EXTRA_INCLUDES + ${AWS_AUTH_INCLUDE_DIR} + DEPENDENCIES + paimon_shared + CURL::libcurl + STATIC_LINK_LIBS + CURL::libcurl + aws_auth_minimal + fmt + SHARED_LINK_LIBS + paimon_shared + SHARED_LINK_FLAGS + ${PAIMON_VERSION_SCRIPT_FLAGS}) + + add_dependencies(paimon_s3_file_system_objlib aws-c-auth_ep) + + if(PAIMON_BUILD_TESTS) + add_paimon_test(s3_file_system_test + SOURCES + s3_file_system_test.cpp + EXTRA_INCLUDES + ${AWS_AUTH_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} + ${PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS} + ${GTEST_LINK_TOOLCHAIN}) + endif() +endif() diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp new file mode 100644 index 00000000..4d896680 --- /dev/null +++ b/src/paimon/fs/s3/s3_file_system.cpp @@ -0,0 +1,1000 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/fs/s3/s3_file_system.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/executor.h" + +namespace paimon::s3 { +namespace { + +Result PercentEncode(std::string_view value, bool preserve_slash) { + if (value.size() > std::numeric_limits::max()) { + return Status::IOError("S3 URL component is too large to encode"); + } + char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size())); + if (encoded == nullptr) { + return Status::IOError("failed to URL encode S3 component"); + } + ScopeGuard free_encoded([encoded] { curl_free(encoded); }); + std::string result(encoded); + if (preserve_slash) { + result = StringUtils::Replace(result, "%2F", "/"); + } + return result; +} + +Result PercentDecode(std::string_view value, const std::string& field) { + for (size_t position = 0; position < value.size(); ++position) { + if (value[position] == '%' && + (position + 2 >= value.size() || + !std::isxdigit(static_cast(value[position + 1])) || + !std::isxdigit(static_cast(value[position + 2])))) { + return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); + } + } + if (value.size() > std::numeric_limits::max()) { + return Status::IOError(fmt::format("S3 {} is too large to URL decode", field)); + } + int decoded_size = 0; + char* decoded = + curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size); + if (decoded == nullptr) { + return Status::IOError(fmt::format("failed to URL decode S3 {}", field)); + } + ScopeGuard free_decoded([decoded] { curl_free(decoded); }); + std::string result(decoded, decoded_size); + return result; +} + +Result ParseNonNegativeInt64(const std::string& value, const std::string& field) { + std::optional result = StringUtils::StringToValue(value); + if (!result || *result < 0 || (!value.empty() && value.front() == '-')) { + return Status::IOError(fmt::format("S3 {} is not a non-negative integer", field)); + } + return *result; +} + +int64_t ParseModificationTime(const std::string& value) { + time_t seconds = curl_getdate(value.c_str(), nullptr); + return seconds == static_cast(-1) ? 0 : static_cast(seconds) * 1000; +} + +std::string XmlUnescape(const std::string& value) { + const std::pair entities[] = { + {"&", "&"}, {"<", "<"}, {">", ">"}, {""", "\""}, {"'", "'"}}; + + std::string result; + result.reserve(value.size()); + for (size_t position = 0; position < value.size();) { + bool matched = false; + if (value[position] == '&') { + for (const auto& [entity, replacement] : entities) { + size_t entity_size = std::strlen(entity); + if (value.compare(position, entity_size, entity) == 0) { + result.append(replacement); + position += entity_size; + matched = true; + break; + } + } + } + if (!matched) { + result.push_back(value[position++]); + } + } + return result; +} + +std::optional TagValue(const std::string& xml, const std::string& tag, + size_t offset = 0) { + std::string begin = "<" + tag + ">"; + std::string end = ""; + size_t begin_position = xml.find(begin, offset); + if (begin_position == std::string::npos) { + return std::nullopt; + } + begin_position += begin.size(); + size_t end_position = xml.find(end, begin_position); + if (end_position == std::string::npos) { + return std::nullopt; + } + return XmlUnescape(xml.substr(begin_position, end_position - begin_position)); +} + +Result> TagBlocks(const std::string& xml, const std::string& tag) { + std::vector blocks; + std::string begin = "<" + tag + ">"; + std::string end = ""; + size_t position = 0; + while (true) { + size_t begin_position = xml.find(begin, position); + size_t unexpected_end = xml.find(end, position); + if (begin_position == std::string::npos) { + if (unexpected_end != std::string::npos) { + return Status::IOError(fmt::format("malformed S3 XML element {}", tag)); + } + break; + } + if (unexpected_end != std::string::npos && unexpected_end < begin_position) { + return Status::IOError(fmt::format("malformed S3 XML element {}", tag)); + } + size_t end_position = xml.find(end, begin_position + begin.size()); + if (end_position == std::string::npos) { + return Status::IOError(fmt::format("malformed S3 XML element {}", tag)); + } + size_t nested_begin = xml.find(begin, begin_position + begin.size()); + if (nested_begin != std::string::npos && nested_begin < end_position) { + return Status::IOError(fmt::format("malformed S3 XML element {}", tag)); + } + end_position += end.size(); + blocks.push_back(xml.substr(begin_position, end_position - begin_position)); + position = end_position; + } + return blocks; +} + +class AwsAuthRuntime { + public: + static Result> Create() { + std::unique_ptr runtime(new AwsAuthRuntime()); + PAIMON_RETURN_NOT_OK(runtime->Initialize()); + return runtime; + } + + ~AwsAuthRuntime() { + if (tls_context_ != nullptr) { + aws_tls_ctx_release(tls_context_); + } + if (bootstrap_ != nullptr) { + aws_client_bootstrap_release(bootstrap_); + } + if (resolver_ != nullptr) { + aws_host_resolver_release(resolver_); + } + if (event_loop_group_ != nullptr) { + aws_event_loop_group_release(event_loop_group_); + } + if (library_initialized_) { + aws_auth_library_clean_up(); + } + } + + aws_allocator* allocator() const { + return allocator_; + } + aws_client_bootstrap* bootstrap() const { + return bootstrap_; + } + aws_tls_ctx* tls_context() const { + return tls_context_; + } + + private: + AwsAuthRuntime() : allocator_(aws_default_allocator()) {} + + Status Initialize() { + aws_auth_library_init(allocator_); + library_initialized_ = true; + event_loop_group_ = aws_event_loop_group_new_default(allocator_, 1, nullptr); + if (event_loop_group_ == nullptr) { + return InitializationError("event loop group"); + } + aws_host_resolver_default_options resolver_options{}; + resolver_options.el_group = event_loop_group_; + resolver_options.max_entries = 8; + resolver_ = aws_host_resolver_new_default(allocator_, &resolver_options); + if (resolver_ == nullptr) { + return InitializationError("host resolver"); + } + aws_client_bootstrap_options bootstrap_options{}; + bootstrap_options.event_loop_group = event_loop_group_; + bootstrap_options.host_resolver = resolver_; + bootstrap_ = aws_client_bootstrap_new(allocator_, &bootstrap_options); + if (bootstrap_ == nullptr) { + return InitializationError("client bootstrap"); + } + aws_tls_ctx_options tls_options; + aws_tls_ctx_options_init_default_client(&tls_options, allocator_); + tls_context_ = aws_tls_client_ctx_new(allocator_, &tls_options); + aws_tls_ctx_options_clean_up(&tls_options); + if (tls_context_ == nullptr) { + return InitializationError("TLS context"); + } + return Status::OK(); + } + + Status InitializationError(const std::string& component) const { + return Status::IOError(fmt::format("failed to initialize AWS {}: {}", component, + aws_error_debug_str(aws_last_error()))); + } + + aws_allocator* allocator_; + aws_event_loop_group* event_loop_group_ = nullptr; + aws_host_resolver* resolver_ = nullptr; + aws_client_bootstrap* bootstrap_ = nullptr; + aws_tls_ctx* tls_context_ = nullptr; + bool library_initialized_ = false; +}; + +Result GetAwsAuthRuntime() { + static const Result runtime = [] { + Result> created = AwsAuthRuntime::Create(); + if (!created.ok()) { + return Result(created.status()); + } + return Result(std::move(created).value().release()); + }(); + return runtime; +} + +aws_byte_cursor Cursor(const std::string& value) { + return aws_byte_cursor_from_array(value.data(), value.size()); +} + +const char* CanonicalS3OptionName(const std::string& option) { + if (option == "access-key" || option == "access.key" || option == "accessKeyId") { + return kS3AccessKeyOption; + } + if (option == "secret-key" || option == "secret.key" || option == "accessKeySecret") { + return kS3SecretKeyOption; + } + if (option == "session.token" || option == "session-token" || option == "security.token" || + option == "security-token" || option == "securityToken") { + return kS3SessionTokenOption; + } + if (option == "endpoint") { + return kS3EndpointOption; + } + if (option == "region") { + return kS3RegionOption; + } + if (option == "path-style-access" || option == "path.style.access") { + return kS3PathStyleAccessOption; + } + if (option == "profile") { + return kS3ProfileOption; + } + return nullptr; +} + +std::map NormalizeS3Options( + const std::map& options) { + std::map normalized = options; + for (const auto& [key, value] : options) { + for (const char* prefix : {"s3a.", "fs.s3.", "fs.s3a."}) { + if (!StringUtils::StartsWith(key, prefix)) { + continue; + } + const char* canonical = CanonicalS3OptionName(key.substr(std::strlen(prefix))); + if (canonical != nullptr && normalized.find(canonical) == normalized.end()) { + normalized.emplace(canonical, value); + } + break; + } + if (StringUtils::StartsWith(key, "s3.")) { + const char* canonical = CanonicalS3OptionName(key.substr(std::strlen("s3."))); + if (canonical != nullptr && normalized.find(canonical) == normalized.end()) { + normalized.emplace(canonical, value); + } + } + } + return normalized; +} + +std::shared_ptr WrapProvider(aws_credentials_provider* provider) { + return std::shared_ptr(provider, aws_credentials_provider_release); +} + +Result ResolveRegion(const std::map& options) { + auto region = options.find(kS3RegionOption); + if (region != options.end() && !region->second.empty()) { + return region->second; + } + const char* environment_region = std::getenv("AWS_REGION"); + if (environment_region != nullptr && environment_region[0] != '\0') { + return std::string(environment_region); + } + environment_region = std::getenv("AWS_DEFAULT_REGION"); + if (environment_region != nullptr && environment_region[0] != '\0') { + return std::string(environment_region); + } + + PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime()); + aws_byte_cursor profile_override{}; + const aws_byte_cursor* profile_override_ptr = nullptr; + auto profile = options.find(kS3ProfileOption); + if (profile != options.end() && !profile->second.empty()) { + profile_override = Cursor(profile->second); + profile_override_ptr = &profile_override; + } + aws_string* config_path = aws_get_config_file_path(runtime->allocator(), nullptr); + aws_string* profile_name = aws_get_profile_name(runtime->allocator(), profile_override_ptr); + aws_profile_collection* profiles = config_path == nullptr + ? nullptr + : aws_profile_collection_new_from_file( + runtime->allocator(), config_path, AWS_PST_CONFIG); + const aws_profile* selected_profile = + profiles == nullptr || profile_name == nullptr + ? nullptr + : aws_profile_collection_get_profile(profiles, profile_name); + aws_string* region_name = aws_string_new_from_c_str(runtime->allocator(), "region"); + const aws_profile_property* property = + selected_profile == nullptr || region_name == nullptr + ? nullptr + : aws_profile_get_property(selected_profile, region_name); + const aws_string* value = + property == nullptr ? nullptr : aws_profile_property_get_value(property); + std::string resolved = value == nullptr ? "" : aws_string_c_str(value); + aws_string_destroy(region_name); + aws_profile_collection_release(profiles); + aws_string_destroy(profile_name); + aws_string_destroy(config_path); + return resolved.empty() ? "us-east-1" : resolved; +} + +Result> MakeCredentialsProvider( + const std::map& options) { + PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime()); + auto access = options.find(kS3AccessKeyOption); + if (access != options.end()) { + const std::string& secret = options.at(kS3SecretKeyOption); + std::string token; + auto configured_token = options.find(kS3SessionTokenOption); + if (configured_token != options.end()) { + token = configured_token->second; + } + aws_credentials_provider_static_options static_options{}; + static_options.access_key_id = Cursor(access->second); + static_options.secret_access_key = Cursor(secret); + static_options.session_token = Cursor(token); + return WrapProvider( + aws_credentials_provider_new_static(runtime->allocator(), &static_options)); + } + + aws_byte_cursor profile_override{}; + auto profile_iter = options.find(kS3ProfileOption); + if (profile_iter != options.end() && !profile_iter->second.empty()) { + profile_override = Cursor(profile_iter->second); + } + std::string region; + auto region_iter = options.find(kS3RegionOption); + if (region_iter != options.end()) { + region = region_iter->second; + } + + std::vector providers; + aws_credentials_provider_environment_options environment_options{}; + providers.push_back( + aws_credentials_provider_new_environment(runtime->allocator(), &environment_options)); + + aws_credentials_provider_profile_options profile_options{}; + profile_options.profile_name_override = profile_override; + profile_options.bootstrap = runtime->bootstrap(); + profile_options.tls_ctx = runtime->tls_context(); + providers.push_back( + aws_credentials_provider_new_profile(runtime->allocator(), &profile_options)); + + aws_credentials_provider_sts_web_identity_options web_options{}; + web_options.profile_name_override = profile_override; + web_options.region = Cursor(region); + web_options.bootstrap = runtime->bootstrap(); + web_options.tls_ctx = runtime->tls_context(); + providers.push_back( + aws_credentials_provider_new_sts_web_identity(runtime->allocator(), &web_options)); + + aws_credentials_provider_sso_options sso_options{}; + sso_options.profile_name_override = profile_override; + sso_options.bootstrap = runtime->bootstrap(); + sso_options.tls_ctx = runtime->tls_context(); + providers.push_back(aws_credentials_provider_new_sso(runtime->allocator(), &sso_options)); + + aws_credentials_provider_login_options login_options{}; + login_options.profile_name_override = profile_override; + login_options.login_region = Cursor(region); + login_options.bootstrap = runtime->bootstrap(); + login_options.tls_ctx = runtime->tls_context(); + providers.push_back(aws_credentials_provider_new_login(runtime->allocator(), &login_options)); + + aws_credentials_provider_chain_default_options default_options{}; + default_options.profile_name_override = profile_override; + default_options.bootstrap = runtime->bootstrap(); + default_options.tls_ctx = runtime->tls_context(); + default_options.skip_environment_credentials_provider = true; + providers.push_back( + aws_credentials_provider_new_chain_default(runtime->allocator(), &default_options)); + + providers.erase(std::remove(providers.begin(), providers.end(), nullptr), providers.end()); + aws_credentials_provider_chain_options chain_options{}; + chain_options.providers = providers.data(); + chain_options.provider_count = providers.size(); + aws_credentials_provider* chain = + aws_credentials_provider_new_chain(runtime->allocator(), &chain_options); + for (aws_credentials_provider* provider : providers) { + aws_credentials_provider_release(provider); + } + if (chain == nullptr) { + return std::shared_ptr(); + } + aws_credentials_provider_cached_options cached_options{}; + cached_options.source = chain; + cached_options.refresh_time_in_milliseconds = 15 * 60 * 1000; + aws_credentials_provider* cached = + aws_credentials_provider_new_cached(runtime->allocator(), &cached_options); + aws_credentials_provider_release(chain); + return WrapProvider(cached); +} + +struct Endpoint { + std::string scheme; + std::string authority; + std::string base_path; +}; + +Result ParseEndpoint(std::string endpoint) { + if (endpoint.find("://") == std::string::npos) { + endpoint = "https://" + endpoint; + } + CURLU* url = curl_url(); + if (url == nullptr) { + return Status::IOError("failed to create S3 endpoint parser"); + } + ScopeGuard cleanup_url([url] { curl_url_cleanup(url); }); + CURLUcode code = curl_url_set(url, CURLUPART_URL, endpoint.c_str(), 0); + if (code != CURLUE_OK) { + return Status::Invalid( + fmt::format("invalid S3 endpoint {}: code {}", endpoint, static_cast(code))); + } + auto get_part = [url, &endpoint](CURLUPart part, CURLUcode no_value, + const char* name) -> Result> { + char* value = nullptr; + CURLUcode result = curl_url_get(url, part, &value, 0); + if (result == no_value) { + return std::optional(); + } + if (result != CURLUE_OK) { + return Status::Invalid(fmt::format("invalid S3 endpoint {} {}: code {}", endpoint, name, + static_cast(result))); + } + ScopeGuard free_value([value] { curl_free(value); }); + return std::optional(value); + }; + PAIMON_ASSIGN_OR_RAISE(std::optional scheme, + get_part(CURLUPART_SCHEME, CURLUE_NO_SCHEME, "scheme")); + PAIMON_ASSIGN_OR_RAISE(std::optional host, + get_part(CURLUPART_HOST, CURLUE_NO_HOST, "host")); + std::string normalized_scheme = scheme ? StringUtils::ToLowerCase(*scheme) : ""; + if (!host || (normalized_scheme != "http" && normalized_scheme != "https")) { + return Status::Invalid(fmt::format("invalid S3 endpoint {}", endpoint)); + } + PAIMON_ASSIGN_OR_RAISE(std::optional user, + get_part(CURLUPART_USER, CURLUE_NO_USER, "user")); + PAIMON_ASSIGN_OR_RAISE(std::optional password, + get_part(CURLUPART_PASSWORD, CURLUE_NO_PASSWORD, "password")); + PAIMON_ASSIGN_OR_RAISE(std::optional query, + get_part(CURLUPART_QUERY, CURLUE_NO_QUERY, "query")); + PAIMON_ASSIGN_OR_RAISE(std::optional fragment, + get_part(CURLUPART_FRAGMENT, CURLUE_NO_FRAGMENT, "fragment")); + if (user || password || query || fragment) { + return Status::Invalid(fmt::format( + "S3 endpoint {} must not contain user, password, query, or fragment", endpoint)); + } + PAIMON_ASSIGN_OR_RAISE(std::optional port, + get_part(CURLUPART_PORT, CURLUE_NO_PORT, "port")); + char* path = nullptr; + code = curl_url_get(url, CURLUPART_PATH, &path, 0); + if (code != CURLUE_OK) { + return Status::Invalid( + fmt::format("invalid S3 endpoint {} path: code {}", endpoint, static_cast(code))); + } + ScopeGuard free_path([path] { curl_free(path); }); + std::string authority = *host; + if (authority.find(':') != std::string::npos) { + authority = "[" + authority + "]"; + } + if (port) { + authority += ":" + *port; + } + return Endpoint{std::move(normalized_scheme), std::move(authority), path}; +} + +bool IsVirtualHostableS3Bucket(const std::string& bucket, bool allow_subdomains) { + if (bucket.size() < 3 || bucket.size() > 63) { + return false; + } + bool label_start = true; + for (size_t index = 0; index < bucket.size(); ++index) { + const auto character = static_cast(bucket[index]); + if (std::islower(character) || std::isdigit(character)) { + label_start = false; + continue; + } + if (character == '-') { + if (label_start || index + 1 == bucket.size() || bucket[index + 1] == '.') { + return false; + } + continue; + } + if (character == '.') { + if (!allow_subdomains || label_start || index + 1 == bucket.size()) { + return false; + } + label_start = true; + continue; + } + return false; + } + return !label_start; +} + +bool IsIpAddressAuthority(const std::string& authority) { + if (!authority.empty() && authority.front() == '[') { + return true; + } + std::string_view host(authority); + size_t port_separator = host.find(':'); + if (port_separator != std::string_view::npos) { + host = host.substr(0, port_separator); + } + size_t component_start = 0; + int component_count = 0; + while (component_start < host.size()) { + size_t component_end = host.find('.', component_start); + std::string_view component = host.substr(component_start, component_end - component_start); + if (component.empty() || component.size() > 3) { + return false; + } + int value = 0; + for (unsigned char character : component) { + if (!std::isdigit(character)) { + return false; + } + value = value * 10 + character - '0'; + } + if (value > 255) { + return false; + } + ++component_count; + if (component_end == std::string_view::npos) { + break; + } + component_start = component_end + 1; + } + return component_count == 4; +} + +const char* AwsDnsSuffixForRegion(const std::string& region) { + if (region.rfind("cn-", 0) == 0) { + return "amazonaws.com.cn"; + } + if (region.rfind("eusc-de-", 0) == 0) { + return "amazonaws.eu"; + } + if (region.rfind("us-iso-", 0) == 0) { + return "c2s.ic.gov"; + } + if (region.rfind("us-isob-", 0) == 0) { + return "sc2s.sgov.gov"; + } + if (region.rfind("eu-isoe-", 0) == 0) { + return "cloud.adc-e.uk"; + } + if (region.rfind("us-isof-", 0) == 0) { + return "csp.hci.ic.gov"; + } + return "amazonaws.com"; +} + +struct SigningContext { + SigningContext(aws_allocator* allocator, aws_http_message* message) + : allocator(allocator), message(message) {} + + aws_allocator* allocator; + aws_http_message* message; + std::mutex mutex; + std::condition_variable condition; + int error_code = AWS_ERROR_SUCCESS; + bool complete = false; +}; + +void OnSigningComplete(aws_signing_result* result, int error_code, void* user_data) { + auto* context = static_cast(user_data); + if (error_code == AWS_ERROR_SUCCESS && + aws_apply_signing_result_to_http_request(context->message, context->allocator, result)) { + error_code = aws_last_error(); + } + { + std::scoped_lock lock(context->mutex); + context->error_code = error_code; + context->complete = true; + } + context->condition.notify_one(); +} + +class S3ObjectStoreClient : public ObjectStoreClient, + public std::enable_shared_from_this { + public: + static Result> Create( + const std::map& options, std::shared_ptr http_client, + std::shared_ptr credentials, std::unique_ptr executor) { + PAIMON_ASSIGN_OR_RAISE(std::string region, ResolveRegion(options)); + auto endpoint = options.find(kS3EndpointOption); + bool use_default_endpoint = endpoint == options.end() || endpoint->second.empty(); + PAIMON_ASSIGN_OR_RAISE( + Endpoint parsed_endpoint, + ParseEndpoint(use_default_endpoint ? fmt::format("https://s3.{}.{}", region, + AwsDnsSuffixForRegion(region)) + : endpoint->second)); + auto path_style = options.find(kS3PathStyleAccessOption); + bool use_path_style = + path_style != options.end() && + OptionsUtils::GetValueFromMap(options, kS3PathStyleAccessOption).value(); + return std::shared_ptr(new S3ObjectStoreClient( + std::move(http_client), std::move(credentials), std::move(executor), + std::move(parsed_endpoint), std::move(region), use_path_style, use_default_endpoint)); + } + + Result HeadObject(const ObjectStorePath& path) const override { + PAIMON_ASSIGN_OR_RAISE(HttpResponse response, + Execute(path, HttpMethod::HEAD, "", {}, nullptr)); + if (response.status_code == 404) { + return Status::NotExist( + fmt::format("s3://{}/{} does not exist", path.bucket, path.key)); + } + PAIMON_RETURN_NOT_OK(CheckResponse(response, "HeadObject", path)); + auto length = response.headers.find("content-length"); + if (length == response.headers.end()) { + return Status::IOError("HeadObject response is missing Content-Length"); + } + int64_t modification_time = 0; + auto modified = response.headers.find("last-modified"); + if (modified != response.headers.end()) { + modification_time = ParseModificationTime(modified->second); + } + PAIMON_ASSIGN_OR_RAISE(int64_t object_size, + ParseNonNegativeInt64(length->second, "Content-Length")); + return ObjectMetadata{path.key, object_size, modification_time}; + } + + Result ListObjects(const ObjectStorePath& path, + const std::string& continuation_token, + int32_t max_keys) const override { + std::string query = "list-type=2&delimiter=%2F&encoding-type=url"; + if (!path.key.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false)); + query += "&prefix=" + encoded_prefix; + } + if (!continuation_token.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::string encoded_token, + PercentEncode(continuation_token, false)); + query += "&continuation-token=" + encoded_token; + } + if (max_keys > 0) { + query += "&max-keys=" + std::to_string(max_keys); + } + std::string body; + HttpBodyConsumer consumer = [&body](const char* data, int64_t size) { + body.append(data, static_cast(size)); + return Status::OK(); + }; + ObjectStorePath bucket_path{path.bucket, ""}; + PAIMON_ASSIGN_OR_RAISE(HttpResponse response, + Execute(bucket_path, HttpMethod::GET, query, {}, consumer)); + if (response.status_code == 404) { + return Status::NotExist(fmt::format("S3 bucket {} does not exist", path.bucket)); + } + PAIMON_RETURN_NOT_OK(CheckResponse(response, "ListObjectsV2", path)); + if (body.find("") == std::string::npos) { + return Status::IOError("malformed S3 ListObjectsV2 XML response"); + } + ListObjectsResult result; + PAIMON_ASSIGN_OR_RAISE(std::vector contents, TagBlocks(body, "Contents")); + for (const std::string& block : contents) { + auto key = TagValue(block, "Key"); + auto size = TagValue(block, "Size"); + if (!key || !size) { + return Status::IOError("S3 ListObjectsV2 Contents is missing Key or Size"); + } + PAIMON_ASSIGN_OR_RAISE(std::string decoded_key, PercentDecode(*key, "Key")); + PAIMON_ASSIGN_OR_RAISE(int64_t object_size, + ParseNonNegativeInt64(*size, "ListObjectsV2 Size")); + int64_t modified = 0; + auto last_modified = TagValue(block, "LastModified"); + if (last_modified) { + modified = ParseModificationTime(*last_modified); + } + result.objects.push_back(ObjectMetadata{decoded_key, object_size, modified}); + } + PAIMON_ASSIGN_OR_RAISE(std::vector common_prefixes, + TagBlocks(body, "CommonPrefixes")); + for (const std::string& block : common_prefixes) { + auto prefix = TagValue(block, "Prefix"); + if (!prefix) { + return Status::IOError("S3 ListObjectsV2 CommonPrefixes is missing Prefix"); + } + PAIMON_ASSIGN_OR_RAISE(std::string decoded_prefix, PercentDecode(*prefix, "Prefix")); + result.common_prefixes.push_back(std::move(decoded_prefix)); + } + auto is_truncated = TagValue(body, "IsTruncated"); + if (!is_truncated || (*is_truncated != "true" && *is_truncated != "false")) { + return Status::IOError("S3 ListObjectsV2 response has invalid IsTruncated"); + } + result.is_truncated = *is_truncated == "true"; + auto token = TagValue(body, "NextContinuationToken"); + if (token) { + result.continuation_token = std::move(*token); + } + return result; + } + + Result GetObjectRange(const ObjectStorePath& path, int64_t offset, int64_t size, + char* buffer) const override { + if (size == 0) { + return 0; + } + int64_t copied = 0; + HttpHeaders headers{{"range", fmt::format("bytes={}-{}", offset, offset + size - 1)}}; + HttpBodyConsumer consumer = [&copied, buffer, size](const char* data, int64_t length) { + if (length > size - copied) { + return Status::IOError("S3 range response exceeds the requested length"); + } + std::memcpy(buffer + copied, data, static_cast(length)); + copied += length; + return Status::OK(); + }; + PAIMON_ASSIGN_OR_RAISE(HttpResponse response, + Execute(path, HttpMethod::GET, "", headers, consumer)); + if (response.status_code == 404) { + return Status::NotExist( + fmt::format("s3://{}/{} does not exist", path.bucket, path.key)); + } + PAIMON_RETURN_NOT_OK(CheckResponse(response, "GetObject", path)); + if (copied != size) { + return Status::IOError( + fmt::format("GetObject read {} bytes for s3://{}/{}, expected {}", copied, + path.bucket, path.key, size)); + } + return copied; + } + + void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size, + char* buffer, std::function&& callback) const override { + auto self = shared_from_this(); + executor_->Add([self = std::move(self), path, offset, size, buffer, + callback = std::move(callback)]() mutable { + Result result = self->GetObjectRange(path, offset, size, buffer); + callback(result.ok() ? Status::OK() : result.status()); + }); + } + + private: + S3ObjectStoreClient(std::shared_ptr http_client, + std::shared_ptr credentials, + std::unique_ptr executor, Endpoint endpoint, std::string region, + bool path_style, bool use_default_endpoint) + : http_client_(std::move(http_client)), + credentials_(std::move(credentials)), + endpoint_(std::move(endpoint)), + region_(std::move(region)), + path_style_(path_style), + use_default_endpoint_(use_default_endpoint), + executor_(std::move(executor)) {} + Status CheckResponse(const HttpResponse& response, const std::string& operation, + const ObjectStorePath& path) const { + if (response.status_code >= 200 && response.status_code < 300) { + return Status::OK(); + } + return Status::IOError(fmt::format("{} failed for s3://{}/{}: HTTP {}", operation, + path.bucket, path.key, response.status_code)); + } + + Result Execute(const ObjectStorePath& object, HttpMethod method, + const std::string& query, const HttpHeaders& headers, + const HttpBodyConsumer& consumer) const { + std::string authority = endpoint_.authority; + std::string request_path = endpoint_.base_path; + if (request_path.empty() || request_path.back() != '/') { + request_path += '/'; + } + bool use_path_style = + path_style_ || + (use_default_endpoint_ + ? !IsVirtualHostableS3Bucket(object.bucket, false) + : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) || + !IsVirtualHostableS3Bucket(object.bucket, true)); + if (use_path_style) { + PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false)); + request_path += encoded_bucket + "/"; + } else { + authority = object.bucket + "." + authority; + } + PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true)); + request_path += encoded_key; + if (!query.empty()) { + request_path += "?" + query; + } + + PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime()); + aws_http_message* message = aws_http_message_new_request(runtime->allocator()); + if (message == nullptr) { + return Status::IOError("failed to create S3 HTTP request"); + } + ScopeGuard release_message([message] { aws_http_message_release(message); }); + std::string method_name = method == HttpMethod::HEAD ? "HEAD" : "GET"; + aws_http_message_set_request_method(message, Cursor(method_name)); + aws_http_message_set_request_path(message, Cursor(request_path)); + aws_http_header host_header{}; + host_header.name = aws_byte_cursor_from_c_str("host"); + host_header.value = Cursor(authority); + aws_http_message_add_header(message, host_header); + for (const auto& [name, value] : headers) { + aws_http_header header{}; + header.name = Cursor(name); + header.value = Cursor(value); + aws_http_message_add_header(message, header); + } + aws_signable* signable = aws_signable_new_http_request(runtime->allocator(), message); + if (signable == nullptr) { + return Status::IOError("failed to create S3 signable request"); + } + ScopeGuard destroy_signable([signable] { aws_signable_destroy(signable); }); + aws_signing_config_aws config{}; + config.config_type = AWS_SIGNING_CONFIG_AWS; + config.algorithm = AWS_SIGNING_ALGORITHM_V4; + config.signature_type = AWS_ST_HTTP_REQUEST_HEADERS; + config.region = Cursor(region_); + config.service = aws_byte_cursor_from_c_str("s3"); + aws_date_time_init_now(&config.date); + config.flags.use_double_uri_encode = false; + config.flags.should_normalize_uri_path = false; + config.signed_body_value = g_aws_signed_body_value_unsigned_payload; + config.signed_body_header = AWS_SBHT_X_AMZ_CONTENT_SHA256; + config.credentials_provider = credentials_.get(); + + SigningContext context(runtime->allocator(), message); + int result = aws_sign_request_aws(runtime->allocator(), signable, + reinterpret_cast(&config), + OnSigningComplete, &context); + if (result == AWS_OP_SUCCESS) { + std::unique_lock lock(context.mutex); + context.condition.wait(lock, [&context] { return context.complete; }); + } + if (result != AWS_OP_SUCCESS || context.error_code != AWS_ERROR_SUCCESS) { + int error = result == AWS_OP_SUCCESS ? context.error_code : aws_last_error(); + return Status::IOError( + fmt::format("failed to sign S3 request: {}", aws_error_debug_str(error))); + } + + HttpRequest request; + request.method = method; + request.url = endpoint_.scheme + "://" + authority + request_path; + aws_http_headers* signed_headers = aws_http_message_get_headers(message); + for (size_t i = 0; i < aws_http_headers_count(signed_headers); ++i) { + aws_http_header header; + aws_http_headers_get_index(signed_headers, i, &header); + request.headers[std::string(reinterpret_cast(header.name.ptr), + header.name.len)] = + std::string(reinterpret_cast(header.value.ptr), header.value.len); + } + HttpBodyConsumer body_consumer = consumer; + if (!body_consumer) { + body_consumer = [](const char*, int64_t) { return Status::OK(); }; + } + return http_client_->Execute(request, body_consumer); + } + + std::shared_ptr http_client_; + std::shared_ptr credentials_; + Endpoint endpoint_; + std::string region_; + bool path_style_ = false; + bool use_default_endpoint_ = false; + std::unique_ptr executor_; +}; + +} // namespace + +Status ValidateS3Options(const std::map& options) { + auto access = options.find(kS3AccessKeyOption); + auto secret = options.find(kS3SecretKeyOption); + auto token = options.find(kS3SessionTokenOption); + bool has_access = access != options.end(); + bool has_secret = secret != options.end(); + bool has_token = token != options.end(); + if (has_access != has_secret) { + return Status::Invalid(fmt::format("{} and {} must be configured together", + kS3AccessKeyOption, kS3SecretKeyOption)); + } + if (has_token && !has_access) { + return Status::Invalid(fmt::format("{} requires {} and {}", kS3SessionTokenOption, + kS3AccessKeyOption, kS3SecretKeyOption)); + } + if (has_access && access->second.empty()) { + return Status::Invalid(fmt::format("{} must not be empty", kS3AccessKeyOption)); + } + if (has_secret && secret->second.empty()) { + return Status::Invalid(fmt::format("{} must not be empty", kS3SecretKeyOption)); + } + if (options.find(kS3PathStyleAccessOption) == options.end()) { + return Status::OK(); + } + Result parsed = OptionsUtils::GetValueFromMap(options, kS3PathStyleAccessOption); + if (!parsed.ok()) { + return Status::Invalid( + fmt::format("{} {}", kS3PathStyleAccessOption, parsed.status().message())); + } + return Status::OK(); +} + +S3FileSystem::S3FileSystem(std::shared_ptr client) + : ObjectStoreFileSystem("s3", std::move(client)) {} + +Result> S3FileSystem::Create( + const std::map& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr client, + MakeS3ObjectStoreClient(options, std::make_shared())); + return std::unique_ptr(new S3FileSystem(std::move(client))); +} + +Result> MakeS3ObjectStoreClient( + const std::map& options, std::shared_ptr http_client) { + std::map normalized_options = NormalizeS3Options(options); + PAIMON_RETURN_NOT_OK(ValidateS3Options(normalized_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr credentials, + MakeCredentialsProvider(normalized_options)); + if (!credentials) { + return Status::IOError("failed to initialize S3 credentials provider"); + } + std::unique_ptr executor = CreateDefaultExecutor(); + return S3ObjectStoreClient::Create(normalized_options, std::move(http_client), + std::move(credentials), std::move(executor)); +} + +} // namespace paimon::s3 diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h new file mode 100644 index 00000000..12caf8b7 --- /dev/null +++ b/src/paimon/fs/s3/s3_file_system.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/fs/http_client.h" +#include "paimon/common/fs/object_store_file_system.h" + +namespace paimon::s3 { + +inline constexpr char kS3RegionOption[] = "s3.region"; +inline constexpr char kS3EndpointOption[] = "s3.endpoint"; +inline constexpr char kS3PathStyleAccessOption[] = "s3.path-style-access"; +inline constexpr char kS3ProfileOption[] = "s3.profile"; +inline constexpr char kS3AccessKeyOption[] = "s3.access-key"; +inline constexpr char kS3SecretKeyOption[] = "s3.secret-key"; +inline constexpr char kS3SessionTokenOption[] = "s3.session.token"; + +Status ValidateS3Options(const std::map& options); +Result> MakeS3ObjectStoreClient( + const std::map& options, std::shared_ptr http_client); + +class S3FileSystem : public ObjectStoreFileSystem { + public: + using ObjectStoreFileSystem::Create; + + static Result> Create( + const std::map& options); + ~S3FileSystem() override = default; + + private: + explicit S3FileSystem(std::shared_ptr client); +}; + +} // namespace paimon::s3 diff --git a/src/paimon/fs/s3/s3_file_system_factory.cpp b/src/paimon/fs/s3/s3_file_system_factory.cpp new file mode 100644 index 00000000..8a28e274 --- /dev/null +++ b/src/paimon/fs/s3/s3_file_system_factory.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/fs/s3/s3_file_system_factory.h" + +#include "paimon/factories/factory.h" +#include "paimon/fs/s3/s3_file_system.h" + +namespace paimon::s3 { + +const char S3FileSystemFactory::IDENTIFIER[] = "s3"; + +Result> S3FileSystemFactory::Create( + const std::string&, const std::map& options) const { + return S3FileSystem::Create(options); +} + +REGISTER_PAIMON_FACTORY(S3FileSystemFactory); + +} // namespace paimon::s3 diff --git a/src/paimon/fs/s3/s3_file_system_factory.h b/src/paimon/fs/s3/s3_file_system_factory.h new file mode 100644 index 00000000..189317cd --- /dev/null +++ b/src/paimon/fs/s3/s3_file_system_factory.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include "paimon/fs/file_system_factory.h" + +namespace paimon::s3 { + +class S3FileSystemFactory : public FileSystemFactory { + public: + static const char IDENTIFIER[]; + const char* Identifier() const override { + return IDENTIFIER; + } + Result> Create( + const std::string& path, const std::map& options) const override; +}; + +} // namespace paimon::s3 diff --git a/src/paimon/fs/s3/s3_file_system_test.cpp b/src/paimon/fs/s3/s3_file_system_test.cpp new file mode 100644 index 00000000..09b1bb4b --- /dev/null +++ b/src/paimon/fs/s3/s3_file_system_test.cpp @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/fs/s3/s3_file_system.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/string_utils.h" +#include "paimon/fs/s3/s3_file_system_factory.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::s3 { +namespace { + +class MockHttpClient : public HttpClient { + public: + Result Execute(const HttpRequest& request, + const HttpBodyConsumer& consumer) const override { + request_ = request; + HttpResponse response; + response.status_code = status_code_; + response.headers = response_headers_; + if (!body_.empty()) { + PAIMON_RETURN_NOT_OK(consumer(body_.data(), body_.size())); + response.body_size = body_.size(); + } + return response; + } + + mutable HttpRequest request_; + int32_t status_code_ = 200; + HttpHeaders response_headers_; + std::string body_; +}; + +class ScopedEnvironmentVariable { + public: + ScopedEnvironmentVariable(const char* name, std::optional value) : name_(name) { + const char* previous = std::getenv(name); + if (previous != nullptr) { + previous_ = previous; + } + if (value) { + setenv(name, value->c_str(), 1); + } else { + unsetenv(name); + } + } + + ~ScopedEnvironmentVariable() { + if (previous_) { + setenv(name_.c_str(), previous_->c_str(), 1); + } else { + unsetenv(name_.c_str()); + } + } + + private: + std::string name_; + std::optional previous_; +}; + +std::map StaticOptions() { + return {{kS3AccessKeyOption, "access"}, + {kS3SecretKeyOption, "secret"}, + {kS3SessionTokenOption, "token"}, + {kS3RegionOption, "ap-northeast-2"}}; +} + +const std::string* FindHeader(const HttpHeaders& headers, const std::string& name) { + std::string normalized_name = StringUtils::ToLowerCase(name); + for (const auto& [header_name, value] : headers) { + if (StringUtils::ToLowerCase(header_name) == normalized_name) { + return &value; + } + } + return nullptr; +} + +TEST(S3ObjectStoreClientTest, TestHeadAndSigning) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + ASSERT_OK_AND_ASSIGN(auto metadata, client->HeadObject({"bucket", "a b/file"})); + ASSERT_EQ(metadata.size, 12); + ASSERT_EQ(http->request_.url, "https://bucket.s3.ap-northeast-2.amazonaws.com/a%20b/file"); + const std::string* authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("/ap-northeast-2/s3/aws4_request"), std::string::npos); + ASSERT_NE(authorization->find( + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token"), + std::string::npos); + ASSERT_EQ(http->request_.headers["host"], "bucket.s3.ap-northeast-2.amazonaws.com"); + const std::string* session_token = FindHeader(http->request_.headers, "x-amz-security-token"); + ASSERT_NE(session_token, nullptr); + ASSERT_EQ(*session_token, "token"); +} + +TEST(S3ObjectStoreClientTest, TestDottedBucketUsesPathStyleForDefaultEndpoint) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + ASSERT_OK(client->HeadObject({"paimon.prod.data", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/paimon.prod.data/file"); + ASSERT_EQ(http->request_.headers["host"], "s3.ap-northeast-2.amazonaws.com"); + + auto path_style_options = StaticOptions(); + path_style_options[kS3PathStyleAccessOption] = "false"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr path_style_client, + MakeS3ObjectStoreClient(path_style_options, http)); + ASSERT_OK(path_style_client->HeadObject({"paimon.prod.data", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/paimon.prod.data/file"); +} + +TEST(S3ObjectStoreClientTest, TestCustomEndpointAddressing) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + + auto https_options = StaticOptions(); + https_options[kS3EndpointOption] = "https://s3.example.com"; + https_options[kS3PathStyleAccessOption] = "false"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr https_client, + MakeS3ObjectStoreClient(https_options, http)); + ASSERT_OK(https_client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.example.com/bucket/file"); + ASSERT_OK(https_client->HeadObject({"paimon.prod.data", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.example.com/paimon.prod.data/file"); + + auto http_options = StaticOptions(); + http_options[kS3EndpointOption] = "http://s3.example.com"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr http_client, + MakeS3ObjectStoreClient(http_options, http)); + ASSERT_OK(http_client->HeadObject({"paimon.prod.data", "file"})); + ASSERT_EQ(http->request_.url, "http://paimon.prod.data.s3.example.com/file"); + + auto ip_options = StaticOptions(); + ip_options[kS3EndpointOption] = "http://127.0.0.1:9000"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr ip_client, + MakeS3ObjectStoreClient(ip_options, http)); + ASSERT_OK(ip_client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "http://127.0.0.1:9000/bucket/file"); + + auto base_path_options = StaticOptions(); + base_path_options[kS3EndpointOption] = "HTTPS://s3.example.com/storage"; + base_path_options[kS3PathStyleAccessOption] = "true"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr base_path_client, + MakeS3ObjectStoreClient(base_path_options, http)); + ASSERT_OK(base_path_client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.example.com/storage/bucket/file"); +} + +TEST(S3ObjectStoreClientTest, TestInvalidCustomEndpoint) { + for (const char* endpoint : + {"ftp://s3.example.com", "https://user@s3.example.com", "https://s3.example.com?query", + "https://s3.example.com#fragment"}) { + auto options = StaticOptions(); + options[kS3EndpointOption] = endpoint; + ASSERT_NOK(MakeS3ObjectStoreClient(options, std::make_shared())); + } +} + +TEST(S3ObjectStoreClientTest, TestNonVirtualHostableBucketUsesPathStyle) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + ASSERT_OK(client->HeadObject({"aa", "file"})); + ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/aa/file"); + + auto options = StaticOptions(); + options[kS3EndpointOption] = "http://s3.example.com"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr custom_client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(custom_client->HeadObject({"BucketName", "file"})); + ASSERT_EQ(http->request_.url, "http://s3.example.com/BucketName/file"); +} + +TEST(S3ObjectStoreClientTest, TestPathStyleOptionUsesCommonBooleanParser) { + for (const auto& [value, path_style] : + std::vector>{{"t", true}, + {"y", true}, + {"yes", true}, + {"1", true}, + {"f", false}, + {"n", false}, + {"no", false}, + {"0", false}}) { + auto options = StaticOptions(); + options[kS3PathStyleAccessOption] = value; + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, path_style + ? "https://s3.ap-northeast-2.amazonaws.com/bucket/file" + : "https://bucket.s3.ap-northeast-2.amazonaws.com/file"); + } +} + +TEST(S3ObjectStoreClientTest, TestOptionAliases) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + std::map python_options{ + {"fs.s3.accessKeyId", "python-access"}, {"fs.s3.accessKeySecret", "python-secret"}, + {"fs.s3.securityToken", "python-token"}, {"fs.s3.endpoint", "http://s3.example.com"}, + {"fs.s3.region", "us-west-2"}, {"fs.s3.path.style.access", "true"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr python_client, + MakeS3ObjectStoreClient(python_options, http)); + ASSERT_OK(python_client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "http://s3.example.com/bucket/file"); + const std::string* authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("Credential=python-access/"), std::string::npos); + const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token"); + ASSERT_NE(token, nullptr); + ASSERT_EQ(*token, "python-token"); + + std::map java_options{{"s3a.access.key", "java-access"}, + {"s3a.secret.key", "java-secret"}, + {"s3a.session.token", "java-token"}, + {"s3a.region", "eu-west-1"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr java_client, + MakeS3ObjectStoreClient(java_options, http)); + ASSERT_OK(java_client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "https://bucket.s3.eu-west-1.amazonaws.com/file"); + authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("Credential=java-access/"), std::string::npos); + token = FindHeader(http->request_.headers, "x-amz-security-token"); + ASSERT_NE(token, nullptr); + ASSERT_EQ(*token, "java-token"); +} + +TEST(S3ObjectStoreClientTest, TestCanonicalOptionsTakePrecedenceOverAliases) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + std::map options{{kS3AccessKeyOption, "canonical-access"}, + {kS3SecretKeyOption, "canonical-secret"}, + {"s3a.access.key", "alias-access"}, + {"s3a.secret.key", "alias-secret"}, + {kS3RegionOption, "ap-northeast-2"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + const std::string* authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("Credential=canonical-access/"), std::string::npos); +} + +TEST(S3ObjectStoreClientTest, TestInvalidContentLength) { + for (const std::string content_length : + {"", "invalid", "-1", "12abc", "999999999999999999999999"}) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = content_length; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + ASSERT_TRUE(client->HeadObject({"bucket", "file"}).status().IsIOError()); + } +} + +TEST(S3ObjectStoreClientTest, TestInvalidModificationTime) { + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + http->response_headers_["last-modified"] = "invalid"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + ASSERT_OK_AND_ASSIGN(auto metadata, client->HeadObject({"bucket", "file"})); + ASSERT_EQ(metadata.modification_time, 0); + + http->body_ = + "falsefile" + "invalid12"; + ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", ""}, "", 0)); + ASSERT_EQ(result.objects[0].modification_time, 0); +} + +TEST(S3ObjectStoreClientTest, TestRegionFromEnvironment) { + ScopedEnvironmentVariable region("AWS_REGION", "eu-west-1"); + auto options = StaticOptions(); + options.erase(kS3RegionOption); + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "https://bucket.s3.eu-west-1.amazonaws.com/file"); +} + +TEST(S3ObjectStoreClientTest, TestDefaultEndpointsForAwsPartitions) { + for (const auto& [region, endpoint] : std::vector>{ + {"ap-northeast-2", "https://bucket.s3.ap-northeast-2.amazonaws.com/file"}, + {"cn-north-1", "https://bucket.s3.cn-north-1.amazonaws.com.cn/file"}, + {"eusc-de-east-1", "https://bucket.s3.eusc-de-east-1.amazonaws.eu/file"}, + {"us-iso-east-1", "https://bucket.s3.us-iso-east-1.c2s.ic.gov/file"}, + {"us-isob-east-1", "https://bucket.s3.us-isob-east-1.sc2s.sgov.gov/file"}, + {"eu-isoe-west-1", "https://bucket.s3.eu-isoe-west-1.cloud.adc-e.uk/file"}, + {"us-isof-south-1", "https://bucket.s3.us-isof-south-1.csp.hci.ic.gov/file"}, + {"us-gov-west-1", "https://bucket.s3.us-gov-west-1.amazonaws.com/file"}}) { + auto options = StaticOptions(); + options[kS3RegionOption] = region; + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, endpoint); + } +} + +TEST(S3ObjectStoreClientTest, TestRegionFromProfile) { + auto test_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_dir); + std::filesystem::path config_path = std::filesystem::path(test_dir->Str()) / "region-config"; + { + std::ofstream config(config_path); + config << "[profile test-profile]\nregion = ap-south-1\n"; + } + ScopedEnvironmentVariable region("AWS_REGION", std::nullopt); + ScopedEnvironmentVariable default_region("AWS_DEFAULT_REGION", std::nullopt); + ScopedEnvironmentVariable config_file("AWS_CONFIG_FILE", config_path.string()); + auto options = StaticOptions(); + options.erase(kS3RegionOption); + options[kS3ProfileOption] = "test-profile"; + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(options, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + ASSERT_EQ(http->request_.url, "https://bucket.s3.ap-south-1.amazonaws.com/file"); +} + +TEST(S3ObjectStoreClientTest, TestCredentialsFromEnvironmentProfile) { + auto test_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_dir); + std::filesystem::path credentials_path = + std::filesystem::path(test_dir->Str()) / "credentials-config"; + { + std::ofstream credentials(credentials_path); + credentials << "[environment-profile]\n" + "aws_access_key_id = profile-access\n" + "aws_secret_access_key = profile-secret\n" + "aws_session_token = profile-token\n" + "[default]\n" + "aws_access_key_id = default-access\n" + "aws_secret_access_key = default-secret\n" + "aws_session_token = default-token\n"; + } + ScopedEnvironmentVariable credentials_file("AWS_SHARED_CREDENTIALS_FILE", + credentials_path.string()); + ScopedEnvironmentVariable access_key("AWS_ACCESS_KEY_ID", std::nullopt); + ScopedEnvironmentVariable secret_key("AWS_SECRET_ACCESS_KEY", std::nullopt); + ScopedEnvironmentVariable session_token("AWS_SESSION_TOKEN", std::nullopt); + { + ScopedEnvironmentVariable profile("AWS_PROFILE", "environment-profile"); + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient({{kS3RegionOption, "ap-northeast-2"}}, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + const std::string* authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("Credential=profile-access/"), std::string::npos); + const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token"); + ASSERT_NE(token, nullptr); + ASSERT_EQ(*token, "profile-token"); + } + { + ScopedEnvironmentVariable profile("AWS_PROFILE", std::nullopt); + auto http = std::make_shared(); + http->response_headers_["content-length"] = "12"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient({{kS3RegionOption, "ap-northeast-2"}}, http)); + ASSERT_OK(client->HeadObject({"bucket", "file"})); + const std::string* authorization = FindHeader(http->request_.headers, "authorization"); + ASSERT_NE(authorization, nullptr); + ASSERT_NE(authorization->find("Credential=default-access/"), std::string::npos); + const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token"); + ASSERT_NE(token, nullptr); + ASSERT_EQ(*token, "default-token"); + } +} + +TEST(S3ObjectStoreClientTest, TestRangeAndListObjects) { + auto http = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + http->body_ = "data"; + char buffer[4]; + ASSERT_OK_AND_ASSIGN(auto size, client->GetObjectRange({"bucket", "key"}, 2, 4, buffer)); + ASSERT_EQ(size, 4); + ASSERT_EQ(std::string(buffer, sizeof(buffer)), "data"); + ASSERT_EQ(http->request_.headers["range"], "bytes=2-5"); + + http->body_ = + "true" + "dir/a&b2026-01-01T00:00:00Z" + "7dir/a&lt;b8" + "dir/sub/" + "next token"; + ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", "dir/"}, "old token", 10)); + ASSERT_TRUE(result.is_truncated); + ASSERT_EQ(result.continuation_token, "next token"); + ASSERT_EQ(result.objects[0].key, "dir/a&b"); + ASSERT_EQ(result.objects[1].key, "dir/a<b"); + ASSERT_EQ(result.common_prefixes[0], "dir/sub/"); + ASSERT_NE(http->request_.url.find("amazonaws.com/?list-type=2"), std::string::npos); + ASSERT_NE(http->request_.url.find("encoding-type=url"), std::string::npos); + ASSERT_NE(http->request_.url.find("continuation-token=old%20token"), std::string::npos); +} + +TEST(S3ObjectStoreClientTest, TestUrlEncodedListObjects) { + auto http = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + http->body_ = + "true" + "dir/a%26b%0D7" + "dir/sub%25/" + "next%2Ftoken"; + ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", "dir/"}, "", 0)); + ASSERT_EQ(result.objects[0].key, "dir/a&b\r"); + ASSERT_EQ(result.common_prefixes[0], "dir/sub%/"); + ASSERT_EQ(result.continuation_token, "next%2Ftoken"); + + http->body_ = "false"; + ASSERT_OK(client->ListObjects({"bucket", "dir/"}, result.continuation_token, 0)); + ASSERT_NE(http->request_.url.find("continuation-token=next%252Ftoken"), std::string::npos); + + http->body_ = + "false" + "dir/invalid%27"; + ASSERT_TRUE(client->ListObjects({"bucket", "dir/"}, "", 0).status().IsIOError()); +} + +TEST(S3ObjectStoreClientTest, TestInvalidListObjectsResponse) { + auto http = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr client, + MakeS3ObjectStoreClient(StaticOptions(), http)); + for (const std::string size : {"", "abc", "-1", "12abc", "9223372036854775808"}) { + http->body_ = + "falsekey" + "" + + size + ""; + ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError()); + } + http->body_ = + "falsekey" + "1"; + ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError()); + http->body_ = + "key1"; + ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError()); +} + +TEST(S3FileSystemFactoryTest, TestOptionValidation) { + S3FileSystemFactory factory; + ASSERT_TRUE( + factory.Create("s3://bucket", {{kS3AccessKeyOption, ""}, {kS3SecretKeyOption, "secret"}}) + .status() + .IsInvalid()); + ASSERT_TRUE( + factory.Create("s3://bucket", {{kS3PathStyleAccessOption, "treu"}}).status().IsInvalid()); + ASSERT_TRUE( + factory.Create("s3://bucket", {{kS3PathStyleAccessOption, "on"}}).status().IsInvalid()); + ASSERT_TRUE( + factory.Create("s3://bucket", {{kS3SessionTokenOption, "token"}}).status().IsInvalid()); + + auto http = std::make_shared(); + ASSERT_TRUE( + MakeS3ObjectStoreClient({{kS3AccessKeyOption, "access"}}, http).status().IsInvalid()); + ASSERT_TRUE( + MakeS3ObjectStoreClient({{kS3PathStyleAccessOption, "treu"}}, http).status().IsInvalid()); +} + +} // namespace +} // namespace paimon::s3 diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 8ffcc59d..43d85b3a 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -38,7 +38,6 @@ if(PAIMON_ENABLE_LUCENE) lucene arrow fmt - dl Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -64,10 +63,8 @@ if(PAIMON_ENABLE_LUCENE) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_lucene_index_static - "-Wl,--no-whole-archive" + ${PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index e5081386..39cecec2 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -85,6 +85,21 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb } } +void JiebaTokenizer::NormalizeCase(std::string* term) { + bool is_alphanumeric = true; + for (const auto& c : *term) { + if (!std::isalnum(static_cast(c))) { + is_alphanumeric = false; + break; + } + } + if (is_alphanumeric && !term->empty()) { + std::transform(term->begin(), term->end(), term->begin(), [](char ch) { + return static_cast(std::tolower(static_cast(ch))); + }); + } +} + void JiebaTokenizer::Normalize(const std::unordered_set& stop_words, std::vector* input_ptr, std::vector* output_ptr) { @@ -100,19 +115,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words if (stop_words.find(term) != stop_words.end()) { continue; } - // to lower case - bool is_alphanumeric = true; - for (const auto& c : term) { - if (!std::isalnum(static_cast(c))) { - is_alphanumeric = false; - break; - } - } - if (is_alphanumeric && !term.empty()) { - std::transform(term.begin(), term.end(), term.begin(), [](char ch) { - return static_cast(std::tolower(static_cast(ch))); - }); - } + NormalizeCase(&term); output.emplace_back(term.data(), term.length()); } } diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index 6a179df5..2af534ac 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -55,6 +55,8 @@ class JiebaTokenizer : public Lucene::Tokenizer { static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, const std::string& str, std::vector* terms_ptr); + static void NormalizeCase(std::string* term); + // In-place converts each string in `input` to lowercase to avoid data copying. static void Normalize(const std::unordered_set& stop_words, std::vector* input, std::vector* output); diff --git a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp index c84f4b50..fcdbcf7d 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp @@ -109,6 +109,24 @@ TEST_P(JiebaAnalyzerTest, TestNormalize) { ASSERT_EQ(expected, results); } +TEST(JiebaTokenizerTest, TestNormalizeCase) { + std::string alphanumeric_term = "THIS123"; + JiebaTokenizer::NormalizeCase(&alphanumeric_term); + ASSERT_EQ(alphanumeric_term, "this123"); + + std::string mixed_ascii_cjk_term = "B超"; + JiebaTokenizer::NormalizeCase(&mixed_ascii_cjk_term); + ASSERT_EQ(mixed_ascii_cjk_term, "B超"); + + std::string term_with_underscore = "THIS_IS"; + JiebaTokenizer::NormalizeCase(&term_with_underscore); + ASSERT_EQ(term_with_underscore, "THIS_IS"); + + std::string chinese_term = "机器"; + JiebaTokenizer::NormalizeCase(&chinese_term); + ASSERT_EQ(chinese_term, "机器"); +} + INSTANTIATE_TEST_SUITE_P(ReadBufferSize, JiebaAnalyzerTest, ::testing::ValuesIn(std::vector({2, 5, 10, 100}))); diff --git a/src/paimon/global_index/lucene/lucene_global_index.cpp b/src/paimon/global_index/lucene/lucene_global_index.cpp index fa54d377..03715bcc 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index.cpp @@ -37,6 +37,10 @@ namespace paimon::lucene { LuceneGlobalIndex::LuceneGlobalIndex(const std::map& options) : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} +Result>> LuceneGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> LuceneGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index 9f033ba3..68857c79 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -33,6 +34,8 @@ class LuceneGlobalIndex : public GlobalIndexer { public: explicit LuceneGlobalIndex(const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 603f9021..c037f6a0 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -18,6 +18,7 @@ #include "paimon/global_index/lucene/lucene_global_index_reader.h" #include "arrow/c/bridge.h" +#include "lucene++/DisjunctionMaxQuery.h" #include "lucene++/FileUtils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" @@ -52,8 +53,8 @@ Result> LuceneGlobalIndexReader::Create DataInputStream data_input_stream(paimon_input); PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input_stream.ReadValue()); if (version != kVersion) { - return Status::Invalid(fmt::format("LuceneGlobalIndex not support version {}"), - kVersion); + return Status::Invalid( + fmt::format("LuceneGlobalIndex not support version {}", kVersion)); } PAIMON_ASSIGN_OR_RAISE(int32_t num_files, data_input_stream.ReadValue()); for (int32_t i = 0; i < num_files; i++) { @@ -118,6 +119,25 @@ std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::stri return wterms; } +std::string LuceneGlobalIndexReader::NormalizeWildcardQuery(const std::string& query) { + std::string normalized_query; + normalized_query.reserve(query.size()); + size_t term_begin = 0; + for (size_t i = 0; i <= query.size(); i++) { + if (i != query.size() && query[i] != '*' && query[i] != '?') { + continue; + } + std::string term = query.substr(term_begin, i - term_begin); + JiebaTokenizer::NormalizeCase(&term); + normalized_query.append(term); + if (i != query.size()) { + normalized_query.push_back(query[i]); + } + term_begin = i + 1; + } + return normalized_query; +} + Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL || @@ -155,15 +175,43 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery( Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX); - return Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + auto create_query = [this](const std::string& query) -> Lucene::QueryPtr { + return Lucene::newLucene( + Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query))); + }; + std::string normalized_query = full_text_search->query; + JiebaTokenizer::NormalizeCase(&normalized_query); + Lucene::QueryPtr query = create_query(full_text_search->query); + if (normalized_query == full_text_search->query) { + return query; + } + + // Preserve the original query for mixed ASCII/CJK indexed terms such as "B超", whose + // complete token is not lowercased by the index analyzer. The normalized alternative still + // matches pure ASCII terms. DisjunctionMax avoids double-counting scores if both match. + auto disjunction = Lucene::newLucene(0.0); + disjunction->add(query); + disjunction->add(create_query(normalized_query)); + return disjunction; } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD); - return Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + auto create_query = [this](const std::string& query) -> Lucene::QueryPtr { + return Lucene::newLucene( + Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query))); + }; + std::string normalized_query = NormalizeWildcardQuery(full_text_search->query); + Lucene::QueryPtr query = create_query(full_text_search->query); + if (normalized_query == full_text_search->query) { + return query; + } + + auto disjunction = Lucene::newLucene(0.0); + disjunction->add(query); + disjunction->add(create_query(normalized_query)); + return disjunction; } Result> LuceneGlobalIndexReader::SearchWithLimit( @@ -210,6 +258,17 @@ std::shared_ptr LuceneGlobalIndexReader::SearchWithNoLimit( Result> LuceneGlobalIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { + if (!full_text_search) { + return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer"); + } + if (full_text_search->min_score.has_value()) { + // The lucene backend does not support min_score pushdown. Fail loudly + // instead of silently ignoring the threshold and returning unfiltered + // results, which would be a correctness bug for the caller. + return Status::NotImplemented( + "lucene full-text search does not support min_score; " + "min_score pushdown is only available on the tantivy backend"); + } try { Lucene::QueryPtr query; switch (full_text_search->search_type) { diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index b86c332f..4fd8051e 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -128,6 +128,8 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { std::vector TokenizeQuery(const std::string& query) const; + static std::string NormalizeWildcardQuery(const std::string& query); + std::shared_ptr CreateAllResult() const { return nullptr; } diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index a65d630b..4fac71a3 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -236,6 +236,14 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/std::nullopt))); CheckResult(result, {3l}); } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } // test wildcard query { ASSERT_OK_AND_ASSIGN(auto result, @@ -253,6 +261,22 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/std::nullopt))); CheckResult(result, {3l}); } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*?HIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } // test filter { ASSERT_OK_AND_ASSIGN(auto result, @@ -306,6 +330,18 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/RoaringBitmap64::From({1l, 2l, 3l, 100l})))); CheckResult(result, {2l}); } + // min_score pushdown is not supported by the lucene backend: it must fail + // loudly rather than silently ignore the threshold. + { + auto fts = std::make_shared("f0", + /*limit=*/10, "document", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->min_score = 1.5f; + auto result = lucene_reader->VisitFullTextSearch(fts); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); + } } TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) { @@ -429,6 +465,43 @@ TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) { } } +TEST_P(LuceneGlobalIndexTest, TestMixedAsciiCjkPrefixAndWildcard) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + + std::map options = { + {"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.read.buffer-size", std::to_string(GetParam())}, + {"lucene-fts.jieba.tokenize-mode", "query"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["B超检查"], + ["T恤"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(test_root_dir->Str(), data_type_, options, + array, Range(0, 1), tmp_dir->Str())); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateGlobalIndexReader(test_root_dir->Str(), data_type_, options, meta)); + auto lucene_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(lucene_reader); + + ASSERT_OK_AND_ASSIGN(auto prefix_result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/10, "B", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + CheckResult(prefix_result, {0l}); + + ASSERT_OK_AND_ASSIGN(auto wildcard_result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/10, "*T*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(wildcard_result, {1l}); +} + TEST_F(LuceneGlobalIndexTest, TestInvalidWithoutTmpDir) { auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 2cd52911..14a66584 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -207,10 +207,9 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { static_cast(kDefaultReadBufferSize)); input->readBytes(reinterpret_cast(buffer->data()), /*offset=*/0, static_cast(current_write_size)); - PAIMON_ASSIGN_OR_RAISE( - int32_t actual_write_size, - out->Write(buffer->data(), static_cast(current_write_size))); - if (static_cast(actual_write_size) != current_write_size) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_size, + out->Write(buffer->data(), current_write_size)); + if (actual_write_size != current_write_size) { return Status::Invalid( fmt::format("invalid write, try to write {} while actual write {}", current_write_size, actual_write_size)); diff --git a/src/paimon/global_index/lucene/lucene_input.h b/src/paimon/global_index/lucene/lucene_input.h index 22f60df2..f30acd61 100644 --- a/src/paimon/global_index/lucene/lucene_input.h +++ b/src/paimon/global_index/lucene/lucene_input.h @@ -46,7 +46,7 @@ class LuceneIndexInput : public Lucene::BufferedIndexInput { if (!result.ok()) { throw Lucene::IOException(LuceneUtils::StringToWstring(result.status().ToString())); } - return static_cast(result.value()); + return result.value(); } void close() override { if (is_clone_) { @@ -67,7 +67,7 @@ class LuceneIndexInput : public Lucene::BufferedIndexInput { throw Lucene::IOException( LuceneUtils::StringToWstring(read_result.status().ToString())); } - if (read_result.value() != length) { + if (read_result.value() != static_cast(length)) { throw Lucene::IOException(L"actual read len and expect read len mismatch"); } } diff --git a/src/paimon/global_index/lumina/CMakeLists.txt b/src/paimon/global_index/lumina/CMakeLists.txt index f85af649..b0496df6 100644 --- a/src/paimon/global_index/lumina/CMakeLists.txt +++ b/src/paimon/global_index/lumina/CMakeLists.txt @@ -27,7 +27,6 @@ if(PAIMON_ENABLE_LUMINA) arrow glog fmt - dl Threads::Threads SHARED_LINK_LIBS lumina::interface @@ -46,10 +45,8 @@ if(PAIMON_ENABLE_LUMINA) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_lumina_index_static - "-Wl,--no-whole-archive" + ${PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} lumina::interface ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/global_index/lumina/lumina_api_test.cpp b/src/paimon/global_index/lumina/lumina_api_test.cpp index d92315d3..2c4954b5 100644 --- a/src/paimon/global_index/lumina/lumina_api_test.cpp +++ b/src/paimon/global_index/lumina/lumina_api_test.cpp @@ -136,7 +136,7 @@ class LuminaInterfaceTest : public ::testing::Test { ASSERT_GT(paimon_pool->MaxMemoryUsage(), 0); } - void CheckResult(const std::vector<::lumina::api::LuminaSearcher::SearchHit>& search_result, + void CheckResult(const std::vector<::lumina::api::SearchHit>& search_result, const std::vector<::lumina::core::vector_id_t>& expected_row_ids, const std::vector& expected_distances) const { ASSERT_EQ(search_result.size(), expected_row_ids.size()); diff --git a/src/paimon/global_index/lumina/lumina_file_io_test.cpp b/src/paimon/global_index/lumina/lumina_file_io_test.cpp index 571703c6..39dd1b2b 100644 --- a/src/paimon/global_index/lumina/lumina_file_io_test.cpp +++ b/src/paimon/global_index/lumina/lumina_file_io_test.cpp @@ -28,50 +28,42 @@ class LuminaFileIOTest : public ::testing::Test { }; TEST_F(LuminaFileIOTest, TestSimple) { - auto check_write_and_read = [](uint64_t max_write_size, uint64_t max_read_size) { - std::string content = "Hello World."; - auto dir = paimon::test::UniqueTestDirectory::Create("local"); - auto fs = dir->GetFileSystem(); - std::string index_path = dir->Str() + "/lumina_test.index"; - // write content - ASSERT_OK_AND_ASSIGN(std::shared_ptr out, - fs->Create(index_path, /*overwrite=*/false)); - auto writer = std::make_shared(out); - writer->max_write_size_ = max_write_size; - ASSERT_EQ(writer->GetLength().Value(), 0); - ASSERT_TRUE(writer->Write(content.data(), content.length()).IsOk()); - ASSERT_TRUE(writer->Close().IsOk()); - - // check file exist - ASSERT_OK_AND_ASSIGN(bool exist, fs->Exists(index_path)); - ASSERT_TRUE(exist); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_status, - fs->GetFileStatus(index_path)); - ASSERT_FALSE(file_status->IsDir()); - ASSERT_EQ(file_status->GetLen(), content.length()); + std::string content = "Hello World."; + auto dir = paimon::test::UniqueTestDirectory::Create("local"); + auto fs = dir->GetFileSystem(); + std::string index_path = dir->Str() + "/lumina_test.index"; + // write content + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs->Create(index_path, /*overwrite=*/false)); + auto writer = std::make_shared(out); + ASSERT_EQ(writer->GetLength().Value(), 0); + ASSERT_TRUE(writer->Write(content.data(), content.length()).IsOk()); + ASSERT_TRUE(writer->Close().IsOk()); - // read content - ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); - auto reader = std::make_shared(in); - reader->max_read_size_ = max_read_size; - ASSERT_EQ(reader->GetLength().Value(), content.length()); - ASSERT_EQ(reader->GetPosition().Value(), 0); - std::string read_content(content.size(), 0); - ASSERT_TRUE(reader->Read(read_content.data(), read_content.size()).IsOk()); - ASSERT_EQ(read_content, content); - ASSERT_EQ(reader->GetPosition().Value(), content.size()); + // check file exist + ASSERT_OK_AND_ASSIGN(bool exist, fs->Exists(index_path)); + ASSERT_TRUE(exist); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_status, fs->GetFileStatus(index_path)); + ASSERT_FALSE(file_status->IsDir()); + ASSERT_EQ(file_status->GetLen(), content.length()); - // test seek - ASSERT_TRUE(reader->Seek(2).IsOk()); - std::string read_content2(3, 0); - ASSERT_TRUE(reader->Read(read_content2.data(), read_content2.size()).IsOk()); - ASSERT_EQ(read_content2, "llo"); - ASSERT_EQ(reader->GetPosition().Value(), 5); - ASSERT_TRUE(reader->Close().IsOk()); - }; + // read content + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); + auto reader = std::make_shared(in); + ASSERT_EQ(reader->GetLength().Value(), content.length()); + ASSERT_EQ(reader->GetPosition().Value(), 0); + std::string read_content(content.size(), 0); + ASSERT_TRUE(reader->Read(read_content.data(), read_content.size()).IsOk()); + ASSERT_EQ(read_content, content); + ASSERT_EQ(reader->GetPosition().Value(), content.size()); - check_write_and_read(LuminaFileWriter::kMaxWriteSize, LuminaFileReader::kMaxReadSize); - check_write_and_read(2, 2); + // test seek + ASSERT_TRUE(reader->Seek(2).IsOk()); + std::string read_content2(3, 0); + ASSERT_TRUE(reader->Read(read_content2.data(), read_content2.size()).IsOk()); + ASSERT_EQ(read_content2, "llo"); + ASSERT_EQ(reader->GetPosition().Value(), 5); + ASSERT_TRUE(reader->Close().IsOk()); } TEST_F(LuminaFileIOTest, TestReadAsync) { @@ -84,8 +76,7 @@ TEST_F(LuminaFileIOTest, TestReadAsync) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); auto reader = std::make_shared(in); - auto check_read_result = [&](uint64_t max_read_size, std::string& read_content) { - reader->max_read_size_ = max_read_size; + auto check_read_result = [&](std::string& read_content) { bool read_finished = false; std::promise promise; std::future future = promise.get_future(); @@ -105,12 +96,10 @@ TEST_F(LuminaFileIOTest, TestReadAsync) { ASSERT_EQ(content.substr(0, read_content.size()), read_content); }; - for (auto max_read_size : {1, 2, 3, 4, 5, 10, 100}) { - std::string read_content(content.size(), '\0'); - check_read_result(max_read_size, read_content); - // test read empty - std::string empty_content; - check_read_result(max_read_size, empty_content); - } + std::string read_content(content.size(), '\0'); + check_read_result(read_content); + // test read empty + std::string empty_content; + check_read_result(empty_content); } } // namespace paimon::lumina::test diff --git a/src/paimon/global_index/lumina/lumina_file_reader.h b/src/paimon/global_index/lumina/lumina_file_reader.h index db6d2fbb..0ae11692 100644 --- a/src/paimon/global_index/lumina/lumina_file_reader.h +++ b/src/paimon/global_index/lumina/lumina_file_reader.h @@ -18,13 +18,12 @@ #pragma once -#include -#include +#include #include -#include #include "fmt/format.h" #include "lumina/io/FileReader.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { @@ -34,12 +33,16 @@ class LuminaFileReader : public ::lumina::io::FileReader { ~LuminaFileReader() override = default; ::lumina::core::Result GetLength() const noexcept override { - Result length_result = in_->Length(); + Result length_result = in_->Length(); if (!length_result.ok()) { return ::lumina::core::Result::Err( PaimonToLuminaStatus(length_result.status())); } - return ::lumina::core::Result::Ok(length_result.value()); + Status status = ValidateValueInRange(length_result.value(), "file length"); + if (!status.ok()) { + return ::lumina::core::Result::Err(PaimonToLuminaStatus(status)); + } + return ::lumina::core::Result::Ok(static_cast(length_result.value())); } ::lumina::core::Result GetPosition() const noexcept override { @@ -47,29 +50,36 @@ class LuminaFileReader : public ::lumina::io::FileReader { if (!pos_result.ok()) { return ::lumina::core::Result::Err(PaimonToLuminaStatus(pos_result.status())); } + Status status = ValidateValueInRange(pos_result.value(), "file position"); + if (!status.ok()) { + return ::lumina::core::Result::Err(PaimonToLuminaStatus(status)); + } return ::lumina::core::Result::Ok(static_cast(pos_result.value())); } ::lumina::core::Status Seek(uint64_t position) noexcept override { - return PaimonToLuminaStatus(in_->Seek(position, SeekOrigin::FS_SEEK_SET)); + Status status = ValidateValueInRange(position, "seek position"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + return PaimonToLuminaStatus( + in_->Seek(static_cast(position), SeekOrigin::FS_SEEK_SET)); } ::lumina::core::Status Read(char* data, uint64_t size) noexcept override { - uint64_t total_read_size = 0; - while (total_read_size < size) { - uint64_t current_read_size = std::min(size - total_read_size, max_read_size_); - Result read_result = - in_->Read(data + total_read_size, static_cast(current_read_size)); - if (!read_result.ok()) { - return PaimonToLuminaStatus(read_result.status()); - } - if (static_cast(read_result.value()) != current_read_size) { - return ::lumina::core::Status( - ::lumina::core::ErrorCode::IoError, - fmt::format("expect read len {} mismatch actual read len {}", current_read_size, - read_result.value())); - } - total_read_size += current_read_size; + Status status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + Result read_result = in_->Read(data, static_cast(size)); + if (!read_result.ok()) { + return PaimonToLuminaStatus(read_result.status()); + } + if (read_result.value() != static_cast(size)) { + return ::lumina::core::Status( + ::lumina::core::ErrorCode::IoError, + fmt::format("expect read len {} mismatch actual read len {}", size, + read_result.value())); } return ::lumina::core::Status::Ok(); } @@ -81,50 +91,20 @@ class LuminaFileReader : public ::lumina::io::FileReader { return; } - struct ReadContext { - char* current_data; - uint64_t remaining; - uint64_t current_offset; - std::function final_call_back; - std::shared_ptr in; - }; - - auto ctx = std::make_shared( - ReadContext{data, size, offset, std::move(call_back), in_}); - - // recursive lambda to read next chunk - std::function read_next; - read_next = [ctx, max_read_size = max_read_size_, &read_next]() { - if (ctx->remaining == 0) { - // all done - ctx->final_call_back(::lumina::core::Status::Ok()); - return; - } - - // determine this chunk's size - uint64_t chunk_size = std::min(ctx->remaining, max_read_size); - auto safe_size = static_cast(chunk_size); - - // issue async read for this chunk - ctx->in->ReadAsync(ctx->current_data, safe_size, ctx->current_offset, - [ctx, safe_size, read_next](const Status& status) { - if (!status.ok()) { - // propagate error immediately - ctx->final_call_back(PaimonToLuminaStatus(status)); - return; - } - // advance pointers and counters - ctx->current_data += safe_size; - ctx->current_offset += safe_size; - ctx->remaining -= safe_size; - - // continue with next chunk - read_next(); - }); - }; - - // start the first read - read_next(); + Status status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + call_back(PaimonToLuminaStatus(status)); + return; + } + status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + call_back(PaimonToLuminaStatus(status)); + return; + } + in_->ReadAsync(data, static_cast(size), static_cast(offset), + [call_back = std::move(call_back)](const Status& status) { + call_back(PaimonToLuminaStatus(status)); + }); } ::lumina::core::Status Close() noexcept override { @@ -132,10 +112,6 @@ class LuminaFileReader : public ::lumina::io::FileReader { } private: - static constexpr uint64_t kMaxReadSize = std::numeric_limits::max(); - - private: - uint64_t max_read_size_ = kMaxReadSize; std::shared_ptr in_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_file_writer.h b/src/paimon/global_index/lumina/lumina_file_writer.h index 16323778..a2514cc6 100644 --- a/src/paimon/global_index/lumina/lumina_file_writer.h +++ b/src/paimon/global_index/lumina/lumina_file_writer.h @@ -18,12 +18,11 @@ #pragma once -#include -#include #include #include "fmt/format.h" #include "lumina/io/FileWriter.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { @@ -41,21 +40,19 @@ class LuminaFileWriter : public ::lumina::io::FileWriter { } ::lumina::core::Status Write(const char* data, uint64_t size) noexcept override { - uint64_t total_write_size = 0; - while (total_write_size < size) { - uint64_t current_write_size = std::min(size - total_write_size, max_write_size_); - Result write_result = - out_->Write(data + total_write_size, static_cast(current_write_size)); - if (!write_result.ok()) { - return PaimonToLuminaStatus(write_result.status()); - } - if (static_cast(write_result.value()) != current_write_size) { - return ::lumina::core::Status( - ::lumina::core::ErrorCode::IoError, - fmt::format("expect write len {} mismatch actual write len {}", - current_write_size, write_result.value())); - } - total_write_size += current_write_size; + Status status = ValidateValueInRange(size, "write size"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + Result write_result = out_->Write(data, static_cast(size)); + if (!write_result.ok()) { + return PaimonToLuminaStatus(write_result.status()); + } + if (write_result.value() != static_cast(size)) { + return ::lumina::core::Status( + ::lumina::core::ErrorCode::IoError, + fmt::format("expect write len {} mismatch actual write len {}", size, + write_result.value())); } return ::lumina::core::Status::Ok(); } @@ -69,10 +66,6 @@ class LuminaFileWriter : public ::lumina::io::FileWriter { } private: - static constexpr uint64_t kMaxWriteSize = std::numeric_limits::max(); - - private: - uint64_t max_write_size_ = kMaxWriteSize; std::shared_ptr out_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 2d13aa12..b27cdc7a 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -18,6 +18,10 @@ #include "paimon/global_index/lumina/lumina_global_index.h" +#include +#include +#include +#include #include #include "arrow/c/bridge.h" @@ -29,6 +33,7 @@ #include "lumina/core/Constants.h" #include "lumina/core/Status.h" #include "lumina/core/Types.h" +#include "lumina/extensions/experimental/BuildCombinedExtensionV0.h" #include "paimon/common/global_index/global_index_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -37,6 +42,9 @@ #include "paimon/global_index/lumina/lumina_file_reader.h" #include "paimon/global_index/lumina/lumina_file_writer.h" #include "paimon/global_index/lumina/lumina_utils.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "rapidjson/document.h" namespace paimon::lumina { #define CHECK_NOT_NULL(pointer, error_msg) \ do { \ @@ -45,6 +53,502 @@ namespace paimon::lumina { } \ } while (0) +namespace { +using TagDimensionData = ::lumina::extensions::experimental::TagDimensionData; +using TagFilter = ::lumina::extensions::experimental::TagFilter; +using TagValue = ::lumina::extensions::experimental::TagValue; +using TagValues = ::lumina::extensions::experimental::TagValues; + +Result GetRequiredStringMember(const rapidjson::Value& obj, + const std::string& field_name, + const std::string& tag_label) { + auto iter = obj.FindMember(field_name.c_str()); + if (iter == obj.MemberEnd()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} missing required field: {}", tag_label, field_name)); + } + if (!iter->value.IsString()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} field {} must be string", tag_label, field_name)); + } + return std::string(iter->value.GetString(), iter->value.GetStringLength()); +} + +Result ParseTagField(const rapidjson::Value& obj, const std::string& tag_label) { + if (!obj.IsObject()) { + return Status::Invalid(fmt::format("lumina tag_schema {} must be object", tag_label)); + } + if (obj.MemberCount() != 3) { + return Status::Invalid(fmt::format( + "lumina tag_schema {} must have exactly 3 fields: key_name, type, value_type", + tag_label)); + } + + PAIMON_ASSIGN_OR_RAISE( + std::string key_name, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagKName), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagType), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string value_type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagVType), tag_label)); + + if (key_name.empty()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} key_name must not be empty", tag_label)); + } + LuminaTagField::Type parsed_type; + if (type == std::string(::lumina::core::kExtensionTagTypeEnum)) { + parsed_type = LuminaTagField::Type::ENUM; + } else if (type == std::string(::lumina::core::kExtensionTagTypeRange)) { + parsed_type = LuminaTagField::Type::RANGE; + } else { + return Status::Invalid( + fmt::format("lumina tag_schema {} has unsupported type: {}", tag_label, type)); + } + + LuminaTagField::ValueType parsed_value_type; + if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt32)) { + parsed_value_type = LuminaTagField::ValueType::INT32; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt64)) { + parsed_value_type = LuminaTagField::ValueType::INT64; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeFloat)) { + parsed_value_type = LuminaTagField::ValueType::FLOAT; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeDouble)) { + parsed_value_type = LuminaTagField::ValueType::DOUBLE; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeString)) { + parsed_value_type = LuminaTagField::ValueType::STRING; + } else { + return Status::Invalid(fmt::format("lumina tag_schema {} has unsupported value_type: {}", + tag_label, value_type)); + } + return LuminaTagField{key_name, parsed_type, parsed_value_type}; +} + +Status ValidateTagArrowType(const LuminaTagField& tag_field, + const std::shared_ptr& field_type) { + auto value_type = field_type; + if (auto list_type = std::dynamic_pointer_cast(field_type)) { + value_type = list_type->value_type(); + } + + bool compatible = false; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT32: + compatible = value_type->id() == arrow::Type::INT8 || + value_type->id() == arrow::Type::INT16 || + value_type->id() == arrow::Type::INT32; + break; + case LuminaTagField::ValueType::INT64: + compatible = value_type->id() == arrow::Type::INT64; + break; + case LuminaTagField::ValueType::FLOAT: + compatible = value_type->id() == arrow::Type::FLOAT; + break; + case LuminaTagField::ValueType::DOUBLE: + compatible = value_type->id() == arrow::Type::DOUBLE; + break; + case LuminaTagField::ValueType::STRING: + compatible = value_type->id() == arrow::Type::STRING; + break; + } + if (!compatible) { + return Status::Invalid( + fmt::format("lumina tag field {} type {} is not compatible with tag_schema value_type", + tag_field.name, field_type->ToString())); + } + return Status::OK(); +} + +template +void AppendPrimitiveTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + values->push_back( + static_cast(static_cast(array.get())->Value(index))); +} + +template +Status AppendTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + if (array->IsNull(index)) { + return Status::OK(); + } + + auto validate_array_type = [&](arrow::Type::type expected_type, + const char* value_type_name) -> Status { + if (array->type_id() != expected_type) { + return Status::Invalid(fmt::format("lumina {} tag field has unsupported arrow type {}", + value_type_name, array->type()->ToString())); + } + return Status::OK(); + }; + + if constexpr (std::is_same_v) { + switch (array->type_id()) { + case arrow::Type::INT8: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT16: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT32: + AppendPrimitiveTagValue(array, index, values); + break; + default: + return Status::Invalid( + fmt::format("lumina integer tag field has unsupported arrow type {}", + array->type()->ToString())); + } + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::INT64, "int64")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::FLOAT, "float")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::DOUBLE, "double")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::STRING, "string")); + auto string_array = static_cast(array.get()); + auto view = string_array->GetView(index); + values->emplace_back(view.data(), view.size()); + } else { + return Status::Invalid("lumina tag field has unsupported value type"); + } + return Status::OK(); +} + +template +Status ExtractTagValues(const std::shared_ptr& field_array, int64_t segment_start, + int64_t segment_len, std::vector>* values) { + values->resize(segment_len); + auto list_array = std::dynamic_pointer_cast(field_array); + if (list_array) { + auto child_values = list_array->values(); + for (int64_t i = 0; i < segment_len; i++) { + int64_t row = segment_start + i; + if (list_array->IsNull(row)) { + continue; + } + auto value_start = list_array->value_offset(row); + auto value_end = list_array->value_offset(row + 1); + auto& row_values = (*values)[i]; + row_values.reserve(value_end - value_start); + for (int64_t value_index = value_start; value_index < value_end; value_index++) { + PAIMON_RETURN_NOT_OK(AppendTagValue(child_values, value_index, &row_values)); + } + } + return Status::OK(); + } + + for (int64_t i = 0; i < segment_len; i++) { + PAIMON_RETURN_NOT_OK(AppendTagValue(field_array, segment_start + i, &(*values)[i])); + } + return Status::OK(); +} + +Result LiteralToTagValue(const Literal& literal) { + if (literal.IsNull()) { + return Status::Invalid("lumina tag predicate does not support null literal"); + } + switch (literal.GetType()) { + case FieldType::TINYINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::SMALLINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::INT: + return TagValue(literal.GetValue()); + case FieldType::BIGINT: + return TagValue(literal.GetValue()); + case FieldType::FLOAT: + return TagValue(literal.GetValue()); + case FieldType::DOUBLE: + return TagValue(literal.GetValue()); + case FieldType::STRING: + return TagValue(literal.GetValue()); + default: + return Status::Invalid( + fmt::format("lumina tag predicate does not support literal type {}", + static_cast(literal.GetType()))); + } +} + +Result GetSingleLiteral(const std::vector& literals, + const std::string& function_name) { + if (literals.size() != 1) { + return Status::Invalid( + fmt::format("lumina tag {} predicate requires one literal", function_name)); + } + return &literals[0]; +} + +Result LiteralsToTagValues(const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("lumina tag predicate IN requires at least one literal"); + } + + switch (literals[0].GetType()) { + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::BIGINT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::FLOAT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::DOUBLE: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::STRING: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(std::move(*typed_value)); + } + return TagValues(std::move(values)); + } + default: + return Status::Invalid( + fmt::format("lumina tag predicate IN does not support literal type {}", + static_cast(literals[0].GetType()))); + } +} + +} // namespace + +Result> LuminaIndexWriter::ExtractTagDataForSegment( + const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, int64_t segment_len) { + std::vector tag_dimensions_data; + tag_dimensions_data.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + auto field_array = struct_array->GetFieldByName(tag_field.name); + CHECK_NOT_NULL(field_array, + fmt::format("lumina tag field {} not in input array", tag_field.name)); + + TagDimensionData tag_dimension_data; + tag_dimension_data.tagkName = tag_field.name; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT32: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::INT64: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::FLOAT: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::DOUBLE: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::STRING: { + std::vector> values; + PAIMON_RETURN_NOT_OK(ExtractTagValues(field_array, segment_start, + segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + } + tag_dimensions_data.push_back(std::move(tag_dimension_data)); + } + return tag_dimensions_data; +} + +Result> LuminaGlobalIndex::ParseTagSchema( + const std::map& lumina_options) { + auto iter = lumina_options.find(std::string(::lumina::core::kExtensionTagSchema)); + if (iter == lumina_options.end()) { + return std::vector(); + } + + rapidjson::Document document; + document.Parse(iter->second.c_str()); + if (document.HasParseError()) { + return Status::Invalid("lumina tag_schema must be a valid JSON string"); + } + + std::vector tag_fields; + if (document.IsArray()) { + if (document.Empty()) { + return Status::Invalid("lumina tag_schema must contain at least one tag definition"); + } + tag_fields.reserve(document.Size()); + for (rapidjson::SizeType i = 0; i < document.Size(); i++) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, + ParseTagField(document[i], fmt::format("tag[{}]", i))); + tag_fields.push_back(std::move(field)); + } + } else if (document.IsObject()) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, ParseTagField(document, "tag[0]")); + tag_fields.push_back(std::move(field)); + } else { + return Status::Invalid("lumina tag_schema must be an object or array of objects"); + } + + std::unordered_set seen_names; + for (const auto& field : tag_fields) { + if (!seen_names.insert(field.name).second) { + return Status::Invalid( + fmt::format("lumina tag_schema has duplicate key_name: {}", field.name)); + } + } + return tag_fields; +} + +Status LuminaGlobalIndex::ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields) { + for (const auto& tag_field : tag_fields) { + auto field = struct_type.GetFieldByName(tag_field.name); + CHECK_NOT_NULL( + field, fmt::format("lumina tag field {} not exist in arrow schema", tag_field.name)); + PAIMON_RETURN_NOT_OK(ValidateTagArrowType(tag_field, field->type())); + } + return Status::OK(); +} + +Result<::lumina::extensions::experimental::TagFilter> LuminaIndexReader::PredicateToTagFilter( + const std::shared_ptr& predicate) { + if (!predicate) { + return Status::Invalid("lumina tag predicate must not be null"); + } + + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate) { + std::vector<::lumina::extensions::experimental::TagFilter> children; + children.reserve(compound_predicate->Children().size()); + for (const auto& child : compound_predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(child)); + children.push_back(std::move(tag_filter)); + } + if (children.empty()) { + return Status::Invalid("lumina tag compound predicate must have at least one child"); + } + if (children.size() == 1) { + return std::move(children.front()); + } + switch (compound_predicate->GetFunction().GetType()) { + case Function::Type::AND: + return ::lumina::extensions::experimental::TagFilter::And(std::move(children)); + case Function::Type::OR: + return ::lumina::extensions::experimental::TagFilter::Or(std::move(children)); + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support compound function {}", + compound_predicate->GetFunction().ToString())); + } + } + + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (!leaf_predicate) { + return Status::Invalid( + fmt::format("cannot cast predicate {} to CompoundPredicate or LeafPredicate", + predicate->ToString())); + } + + const auto& literals = leaf_predicate->Literals(); + const auto& field_name = leaf_predicate->FieldName(); + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Eq(field_name, std::move(value)); + } + case Function::Type::GREATER_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gt(field_name, std::move(value)); + } + case Function::Type::GREATER_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gte(field_name, std::move(value)); + } + case Function::Type::LESS_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "less than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lt(field_name, std::move(value)); + } + case Function::Type::LESS_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "less or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lte(field_name, std::move(value)); + } + case Function::Type::IN: { + PAIMON_ASSIGN_OR_RAISE(TagValues values, LiteralsToTagValues(literals)); + return ::lumina::extensions::experimental::TagFilter::In(field_name, std::move(values)); + } + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support leaf function {}", + leaf_predicate->GetFunction().ToString())); + } +} + Result> LuminaGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -67,6 +571,8 @@ Result> LuminaGlobalIndex::CreateWriter( // check options auto lumina_options = OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + PAIMON_RETURN_NOT_OK(ValidateTagFields(*struct_type, tag_fields)); PAIMON_ASSIGN_OR_RAISE(uint32_t dimension, OptionsUtils::GetValueFromMap( lumina_options, std::string(::lumina::core::kDimension))); @@ -78,7 +584,7 @@ Result> LuminaGlobalIndex::CreateWriter( auto lumina_pool = std::make_shared(pool); return std::make_shared( field_name, arrow_type, dimension, file_writer, std::move(builder_options), - ::lumina::api::IOOptions(), lumina_options, lumina_pool); + ::lumina::api::IOOptions(), lumina_options, std::move(tag_fields), lumina_pool); } Result LuminaIndexReader::GetIndexInfo( @@ -113,7 +619,9 @@ Result LuminaIndexReader::GetIndexInfo( return Status::Invalid( fmt::format("invalid distance type {} for lumina", distance_type_str)); } - return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type}); + bool has_tag = lumina_write_options.find(std::string(::lumina::core::kExtensionTagSchema)) != + lumina_write_options.end(); + return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type, has_tag}); } Result> LuminaGlobalIndex::CreateReader( @@ -172,8 +680,30 @@ Result> LuminaGlobalIndex::CreateReader( } auto searcher_with_filter = std::make_unique<::lumina::extensions::SearchWithFilterExtension>(); PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_filter)); + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag; + if (index_info.has_tag) { + searcher_with_tag = + std::make_unique<::lumina::extensions::experimental::SearchWithTagExtension>(); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_tag)); + } return std::make_shared(index_info, std::move(searcher), - std::move(searcher_with_filter), lumina_pool); + std::move(searcher_with_filter), + std::move(searcher_with_tag), lumina_pool); +} + +Result>> LuminaGlobalIndex::GetExtraFieldNames() const { + auto lumina_options = + OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + if (tag_fields.empty()) { + return std::optional>(std::nullopt); + } + std::vector field_names; + field_names.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + field_names.push_back(tag_field.name); + } + return std::optional>(std::move(field_names)); } class LuminaDataset : public ::lumina::api::Dataset { @@ -201,18 +731,67 @@ class LuminaDataset : public ::lumina::api::Dataset { } auto& value_array = array_vec_[cursor_]; int64_t value_array_length = value_array->length(); - int64_t element_count = value_array_length / dimension_; + int64_t batch_element_count = value_array_length / dimension_; const float* value_ptr = value_array->raw_values(); vector_buffer.resize(value_array_length); memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); - id_buffer.resize(element_count); + id_buffer.resize(batch_element_count); std::iota(id_buffer.begin(), id_buffer.end(), static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); // release the array when copy to vector_buffer value_array.reset(); cursor_++; - return ::lumina::core::Result::Ok(static_cast(element_count)); + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); + } + + private: + int64_t element_count_; + uint32_t dimension_; + std::vector> array_vec_; + std::vector start_ids_; + size_t cursor_ = 0; +}; + +class LuminaDatasetWithTag : public ::lumina::extensions::experimental::DatasetWithTag { + public: + LuminaDatasetWithTag(int64_t element_count, uint32_t dimension, + const std::vector>& array_vec, + const std::vector& start_ids, + const std::vector>& tag_data_vec) + : element_count_(element_count), + dimension_(dimension), + array_vec_(array_vec), + start_ids_(start_ids), + tag_data_vec_(tag_data_vec) {} + + uint32_t Dim() const noexcept override { + return dimension_; + } + uint64_t TotalSize() const noexcept override { + return element_count_; + } + + ::lumina::core::Result GetNextBatch( + std::vector& vector_buffer, std::vector<::lumina::core::vector_id_t>& id_buffer, + std::vector& tag_dimensions_data) noexcept override { + if (cursor_ >= array_vec_.size()) { + return ::lumina::core::Result::Ok(0); + } + auto& value_array = array_vec_[cursor_]; + int64_t value_array_length = value_array->length(); + int64_t batch_element_count = value_array_length / dimension_; + const float* value_ptr = value_array->raw_values(); + vector_buffer.resize(value_array_length); + memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); + id_buffer.resize(batch_element_count); + std::iota(id_buffer.begin(), id_buffer.end(), + static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); + tag_dimensions_data = std::move(tag_data_vec_[cursor_]); + + value_array.reset(); + cursor_++; + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); } private: @@ -220,17 +799,16 @@ class LuminaDataset : public ::lumina::api::Dataset { uint32_t dimension_; std::vector> array_vec_; std::vector start_ids_; + std::vector> tag_data_vec_; size_t cursor_ = 0; }; -LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, - const std::shared_ptr& arrow_type, - uint32_t dimension, - const std::shared_ptr& file_manager, - ::lumina::api::BuilderOptions&& builder_options, - ::lumina::api::IOOptions&& io_options, - const std::map& lumina_options, - const std::shared_ptr& pool) +LuminaIndexWriter::LuminaIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + uint32_t dimension, const std::shared_ptr& file_manager, + ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, + const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool) : pool_(pool), field_name_(field_name), arrow_type_(arrow_type), @@ -238,7 +816,8 @@ LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, file_manager_(file_manager), builder_options_(std::move(builder_options)), io_options_(std::move(io_options)), - lumina_options_(lumina_options) {} + lumina_options_(lumina_options), + tag_fields_(std::move(tag_fields)) {} Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) { @@ -285,11 +864,21 @@ Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, return Status::Invalid( "field value array in LuminaIndexWriter is invalid, must not null"); } - if (sliced_values->length() != segment_len * static_cast(dimension_)) { - return Status::Invalid(fmt::format( - "invalid input array in LuminaIndexWriter, length of field array [{}] " - "multiplied dimension [{}] must match length of field value array [{}]", - segment_len, dimension_, sliced_values->length())); + for (int64_t row = segment_start; row < segment_start + segment_len; row++) { + int64_t vector_length = + list_field_array->value_offset(row + 1) - list_field_array->value_offset(row); + if (vector_length != static_cast(dimension_)) { + return Status::Invalid(fmt::format( + "invalid input array in LuminaIndexWriter, vector at row [{}] has length " + "[{}], expected dimension [{}]", + row, vector_length, dimension_)); + } + } + if (!tag_fields_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::vector tag_data, + ExtractTagDataForSegment(struct_array, tag_fields_, + segment_start, segment_len)); + tag_data_vec_.push_back(std::move(tag_data)); } array_vec_.push_back(std::move(sliced_values)); array_start_ids_.push_back(count_ + segment_start); @@ -315,15 +904,25 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.PretrainFrom(dataset1)); // insert data - LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); - std::vector>().swap(array_vec_); - PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + if (tag_fields_.empty()) { + LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); + std::vector>().swap(array_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + } else { + ::lumina::extensions::experimental::BuildWithTagExtension tag_extension; + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.Attach(tag_extension)); + LuminaDatasetWithTag dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_, + tag_data_vec_); + std::vector>().swap(array_vec_); + std::vector>().swap(tag_data_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(tag_extension.InsertFromWithTag(dataset2)); + } // dump index PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_manager_->NewFileName(LuminaDefines::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_manager_->NewOutputStream(index_file_name)) + file_manager_->NewOutputStream(index_file_name)); auto file_writer = std::make_unique(out); PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.Dump(std::move(file_writer), io_options_)); // prepare GlobalIndexIOMeta @@ -340,17 +939,16 @@ LuminaIndexReader::LuminaIndexReader( const LuminaIndexReader::IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& searcher_with_tag, const std::shared_ptr& pool) : index_info_(index_info), pool_(pool), searcher_(std::move(searcher)), - searcher_with_filter_(std::move(searcher_with_filter)) {} + searcher_with_filter_(std::move(searcher_with_filter)), + searcher_with_tag_(std::move(searcher_with_tag)) {} Result> LuminaIndexReader::VisitVectorSearch( const std::shared_ptr& vector_search) { - if (vector_search->predicate) { - return Status::NotImplemented("lumina index not support predicate in VisitVectorSearch"); - } if (vector_search->distance_type && vector_search->distance_type.value() != index_info_.distance_type) { return Status::Invalid("distance type for index and search not match"); @@ -377,7 +975,25 @@ Result> LuminaIndexReader::VisitVectorS ::lumina::api::Query lumina_query(vector_search->query.data(), vector_search->query.size()); ::lumina::api::LuminaSearcher::SearchResult search_result; - if (!vector_search->pre_filter) { + if (vector_search->predicate) { + if (!searcher_with_tag_) { + return Status::Invalid("lumina index was not built with tag"); + } + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(vector_search->predicate)); + if (!vector_search->pre_filter) { + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, searcher_with_tag_->SearchWithTag(lumina_query, tag_filter, + search_options, *pool_)); + } else { + auto lumina_filter = [filter = vector_search->pre_filter]( + ::lumina::core::vector_id_t id) -> bool { return filter(id); }; + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, + searcher_with_tag_->SearchWithTagAndFilter(lumina_query, tag_filter, lumina_filter, + search_options, *pool_)); + } + } else if (!vector_search->pre_filter) { PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(search_result, searcher_->Search(lumina_query, search_options, *pool_)); } else { diff --git a/src/paimon/global_index/lumina/lumina_global_index.h b/src/paimon/global_index/lumina/lumina_global_index.h index 572361a0..c2c30475 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.h +++ b/src/paimon/global_index/lumina/lumina_global_index.h @@ -20,20 +20,44 @@ #include #include +#include #include #include +#include #include #include "arrow/api.h" #include "lumina/api/LuminaSearcher.h" #include "lumina/api/Options.h" #include "lumina/extensions/SearchWithFilterExtension.h" +#include "lumina/extensions/experimental/DatasetWithTag.h" +#include "lumina/extensions/experimental/SearchWithTagExtension.h" +#include "lumina/extensions/experimental/TagFilter.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/lumina/lumina_memory_pool.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { +struct LuminaTagField { + enum class Type { + ENUM, + RANGE, + }; + + enum class ValueType { + INT32, + INT64, + FLOAT, + DOUBLE, + STRING, + }; + + std::string name; + Type type; + ValueType value_type; +}; + /// @note When enabling the lumina global index in `paimon-cpp`, all configuration parameters /// specific to Lumina **must be prefixed with `lumina.`**. /// See `docs/reference/OptionsReference.md` in the Lumina release package for more options. @@ -63,6 +87,8 @@ class LuminaGlobalIndex : public GlobalIndexer { explicit LuminaGlobalIndex(const std::map& options) : options_(options) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -74,6 +100,12 @@ class LuminaGlobalIndex : public GlobalIndexer { const std::shared_ptr& pool) const override; private: + static Result> ParseTagSchema( + const std::map& lumina_options); + + static Status ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields); + std::map options_; }; @@ -85,6 +117,7 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool); Status AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) override; @@ -92,6 +125,11 @@ class LuminaIndexWriter : public GlobalIndexWriter { Result> Finish() override; private: + static Result> + ExtractTagDataForSegment(const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, + int64_t segment_len); + int64_t count_ = 0; int64_t indexed_count_ = 0; std::shared_ptr pool_; @@ -102,8 +140,10 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions builder_options_; ::lumina::api::IOOptions io_options_; std::map lumina_options_; + std::vector tag_fields_; std::vector> array_vec_; std::vector array_start_ids_; + std::vector> tag_data_vec_; }; class LuminaIndexReader : public GlobalIndexReader { @@ -112,11 +152,14 @@ class LuminaIndexReader : public GlobalIndexReader { uint32_t dimension; std::string index_type; VectorSearch::DistanceType distance_type; + bool has_tag; }; LuminaIndexReader( const IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& + searcher_with_tag, const std::shared_ptr& pool); ~LuminaIndexReader() override { @@ -203,9 +246,13 @@ class LuminaIndexReader : public GlobalIndexReader { static Result GetIndexInfo(const GlobalIndexIOMeta& io_meta); private: + static Result<::lumina::extensions::experimental::TagFilter> PredicateToTagFilter( + const std::shared_ptr& predicate); + LuminaIndexReader::IndexInfo index_info_; std::shared_ptr pool_; std::unique_ptr<::lumina::api::LuminaSearcher> searcher_; std::unique_ptr<::lumina::extensions::SearchWithFilterExtension> searcher_with_filter_; + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index 0f314776..2d54950a 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -252,6 +252,443 @@ TEST_F(LuminaGlobalIndexTest, TestWithFilter) { } } +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"({"key_name":"color","type":"enum","value_type":"string"})"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold"], + [[0.0, 1.0, 0.0, 1.0], "warm"], + [[1.0, 0.0, 1.0, 0.0], "cold"], + [[1.0, 1.0, 1.0, 1.0], "warm"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); + } + { + auto pre_filter = [](int64_t id) -> bool { return id < 3; }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered_scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(filtered_scored_result, {1l}, {2.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("price", arrow::float32())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold", 5.0], + [[0.0, 1.0, 0.0, 1.0], "warm", 10.0], + [[1.0, 0.0, 1.0, 0.0], "warm", 20.0], + [[1.0, 1.0, 1.0, 1.0], "warm", 30.0] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr color_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + std::shared_ptr low_price_predicate = PredicateBuilder::LessOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(10.0f)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr price_predicate, + PredicateBuilder::Or({low_price_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({color_predicate, price_predicate})); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + + std::shared_ptr tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category_ids", arrow::list(arrow::int32()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], [1, 2]], + [[0.0, 1.0, 0.0, 1.0], [3, 8]], + [[1.0, 0.0, 1.0, 0.0], [4, 5]], + [[1.0, 1.0, 1.0, 1.0], [6, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::In( + /*field_index=*/1, /*field_name=*/"category_ids", FieldType::INT, {Literal(8), Literal(9)}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"tag_i8","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i16","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i32","type":"range","value_type":"int32"},)" + R"({"key_name":"tag_i64","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_f32","type":"range","value_type":"float"},)" + R"({"key_name":"tag_f64","type":"enum","value_type":"double"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("tag_i8", arrow::int8()), + arrow::field("tag_i16", arrow::int16()), arrow::field("tag_i32", arrow::int32()), + arrow::field("tag_i64", arrow::int64()), arrow::field("tag_f32", arrow::float32()), + arrow::field("tag_f64", arrow::float64())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], 1, 10, 100, 10000000001, 1.5, 1.25], + [[0.0, 1.0, 0.0, 1.0], 2, 20, 200, 10000000002, 2.5, 2.25], + [[1.0, 0.0, 1.0, 0.0], 3, 30, 300, 10000000003, 3.5, 3.25], + [[1.0, 1.0, 1.0, 1.0], 4, 40, 400, 10000000004, 4.5, 4.25] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, expected_ids, expected_scores); + }; + + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"tag_i8", + FieldType::TINYINT, Literal(static_cast(2))), + {1l}, {2.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"tag_i16", FieldType::SMALLINT, + Literal(static_cast(30))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"tag_i32", + FieldType::INT, Literal(250)), + {3l, 2l}, {0.01f, 2.21f}); + search_and_check(PredicateBuilder::Equal( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + Literal(static_cast(10000000003))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + {Literal(static_cast(10000000001)), + Literal(static_cast(10000000004))}), + {3l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/5, /*field_name=*/"tag_f32", + FieldType::FLOAT, Literal(4.0f)), + {3l}, {0.01f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, Literal(3.25)), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, {Literal(2.25), Literal(4.25)}), + {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"enum","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], + [null, "red", ["vip"], 8.0, [0.8], 7, [9]], + [[0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], + [[1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], + [[1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + auto search_and_check_with_filter = + [&](VectorSearch::PreFilter pre_filter, const std::shared_ptr& predicate, + const std::vector& expected_ids, const std::vector& expected_scores) { + std::shared_ptr vector_search = std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + reader->VisitVectorSearch(vector_search)); + CheckResult(scored_result, expected_ids, expected_scores); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected_ids, + expected_scores); + }; + auto search_and_check_error = [&](const std::shared_ptr& predicate, + const std::string& expected_message) { + ASSERT_NOK_WITH_MSG( + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options)), + expected_message); + }; + + search_and_check(/*predicate=*/nullptr, {4l, 2l, 3l, 0l}, {0.01f, 2.01f, 2.21f, 4.21f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + {3l}, {2.21f}); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + {4l}, {0.01f}); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::LessThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {4l, 3l}, {0.01f, 2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"scores", + FieldType::FLOAT, {Literal(0.25f), Literal(0.75f)}), + {4l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/5, /*field_name=*/"category", + FieldType::INT, Literal(7)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"category_ids", + FieldType::INT, {Literal(9)}), + {4l}, {0.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + {}, {}); + search_and_check_error( + PredicateBuilder::NotIn(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + {Literal(FieldType::STRING, "red", 3)}), + "lumina tag predicate does not support leaf function NotIn"); + search_and_check_error( + PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "lumina tag predicate does not support leaf function NotEqual"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/7, /*field_name=*/"unknown", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "unknown tag key 'unknown' in label filter"); + search_and_check_error(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::INT, Literal(1)), + "tag value type mismatch for key 'color'"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"price", FieldType::STRING, + Literal(FieldType::STRING, "x", 1)), + "tag value type mismatch for key 'price'"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, {0l}, + {4.21f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, {}, {}); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/3, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, {0l, 4l}, {4.21f, 0.01f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, {4l}, + {0.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string index_root = test_root_dir->Str(); + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"range","value_type":"string"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'string'"); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"int32"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field color type string is not compatible with tag_schema value_type"); + } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category","type":"enum","value_type":"int32"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field category type int64 is not compatible with tag_schema value_type"); + } + { + std::shared_ptr double_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::float64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"double"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, double_tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'double'"); + } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"int64"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'int64'"); + } +} + +TEST_F(LuminaGlobalIndexTest, TestGetExtraFieldNames) { + { + LuminaGlobalIndex global_index(options_); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_FALSE(field_names); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + LuminaGlobalIndex global_index(tag_options); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_TRUE(field_names); + ASSERT_EQ(field_names.value(), + std::vector({"color", "price", "category_ids"})); + } +} + TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir); @@ -306,8 +743,20 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { .ValueOrDie(); ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), - "invalid input array in LuminaIndexWriter, length of field array [2] multiplied " - "dimension [4] must match length of field value array [7]"); + "invalid input array in LuminaIndexWriter, vector at row [1] has length [3], " + "expected dimension [4]"); + } + { + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, + R"([ + [[0.0, 0.0, 0.0]], + [[0.0, 1.0, 0.0, 1.0, 0.0]] + ])") + .ValueOrDie(); + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), + "invalid input array in LuminaIndexWriter, vector at row [0] has length [3], " + "expected dimension [4]"); } { @@ -374,10 +823,10 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { "f1", /*limit=*/2, query_, /*filter=*/nullptr, PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)), + FieldType::INT, Literal(5)), /*distance_type=*/std::nullopt, /*options=*/std::map())), - "lumina index not support predicate in VisitVectorSearch"); + "lumina index was not built with tag"); } { ASSERT_OK_AND_ASSIGN(auto reader, diff --git a/src/paimon/global_index/lumina/lumina_utils.h b/src/paimon/global_index/lumina/lumina_utils.h index 60f79785..37a81170 100644 --- a/src/paimon/global_index/lumina/lumina_utils.h +++ b/src/paimon/global_index/lumina/lumina_utils.h @@ -36,11 +36,11 @@ namespace paimon::lumina { auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).IsOk(), LuminaToPaimonStatus((result_name).GetStatus()), \ PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).TakeValue(); + lhs = std::move(result_name).TakeValue() #define PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL_FROM_LUMINA( \ - PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)); + PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)) inline ::lumina::core::Status PaimonToLuminaStatus(const Status& status) { switch (status.code()) { diff --git a/src/paimon/global_index/tantivy/CMakeLists.txt b/src/paimon/global_index/tantivy/CMakeLists.txt new file mode 100644 index 00000000..8e1da23f --- /dev/null +++ b/src/paimon/global_index/tantivy/CMakeLists.txt @@ -0,0 +1,268 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tantivy-fulltext global index support library and tests (Rust FFI). + +if(NOT PAIMON_ENABLE_TANTIVY) + return() +endif() + +set(PAIMON_TANTIVY_SUPPORT_SRCS + tantivy_defs.cpp + tantivy_ffi_log.cpp + tantivy_archive_layout.cpp + tantivy_stream_ctx.cpp + tantivy_global_index_writer.cpp + tantivy_global_index_reader.cpp + tantivy_global_index.cpp + tantivy_global_index_factory.cpp) + +add_paimon_lib(paimon_tantivy_support + SOURCES + ${PAIMON_TANTIVY_SUPPORT_SRCS} + DEPENDENCIES + paimon_shared + paimon_tantivy_ffi + STATIC_LINK_LIBS + paimon_tantivy_ffi + arrow + glog + fmt + SHARED_LINK_LIBS + paimon_shared + SHARED_LINK_FLAGS + ${PAIMON_VERSION_SCRIPT_FLAGS}) +# Corrosion's paimon_tantivy_ffi target carries INTERFACE_INCLUDE_DIRECTORIES +# (cbindgen-generated header path). The objlib in add_paimon_lib doesn't link +# against deps,so its compile step misses include dirs.Wire them explicitly. +target_link_libraries(paimon_tantivy_support_objlib PUBLIC paimon_tantivy_ffi) + +# In test builds, bake the jieba dict dir into GetJiebaDictionaryDirFromEnv so +# tests don't have to setenv() process-wide state. Mirrors the lucene module. +if(PAIMON_BUILD_TESTS) + target_compile_definitions(paimon_tantivy_support_objlib + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") +endif() + +if(PAIMON_BUILD_TESTS) + add_paimon_test(tantivy_smoke_test + SOURCES + tantivy_smoke_test.cpp + STATIC_LINK_LIBS + paimon_tantivy_ffi + ${GTEST_LINK_TOOLCHAIN}) + + add_paimon_test(tantivy_ffi_test + SOURCES + tantivy_ffi_test.cpp + STATIC_LINK_LIBS + paimon_shared + "-Wl,--whole-archive" + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + + # Golden-sample tokenizer diff (cppjieba vs jieba-rs). Links against the + # lucene index module to reuse JiebaTokenizer::CutWithMode + Normalize, so it + # can only be built when lucene-fts is enabled (the C++ JiebaTokenizer lives + # in the lucene module). Guarded so the default LUCENE=OFF / TANTIVY=ON build + # doesn't try to link the non-existent paimon_lucene_index_static. + # Note: we mirror the lucene-fts test's link line (see lucene/CMakeLists.txt) + # rather than using the `jieba` imported target, whose INTERFACE_INCLUDE + # concatenates two paths in one string (upstream quirk). + if(PAIMON_ENABLE_LUCENE) + add_paimon_test(tantivy_tokenizer_test + SOURCES + tantivy_tokenizer_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-tokenizer-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}" + PAIMON_TANTIVY_GOLDEN_DIR="${CMAKE_SOURCE_DIR}/test/test_data/tokenizer_golden" + ) + target_include_directories(paimon-tantivy-tokenizer-test SYSTEM + PRIVATE ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) + endif() + + # Writer test. Builds an Arrow batch, runs the writer through + # GlobalIndexFileManager + LocalFileSystem, then validates the packed + # on-disk format. + add_paimon_test(tantivy_writer_test + SOURCES + tantivy_writer_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-writer-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Reader + five query types end-to-end. + add_paimon_test(tantivy_filter_limit_test + SOURCES + tantivy_filter_limit_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-filter-limit-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Java → C++ cross-read test. The fixture is produced by paimon-java's + # `TantivyIndexFixtureGen` and checked in under + # test/test_data/java_tantivy_fixtures/. See that directory's README for + # regeneration instructions. + add_paimon_test(tantivy_java_compat_test + SOURCES + tantivy_java_compat_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-java-compat-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}" + PAIMON_TANTIVY_JAVA_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/java_tantivy_fixtures" + PAIMON_TANTIVY_CPP_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/cpp_tantivy_fixtures" + ) + + # Streaming reader and writer integration coverage: + # ParseArchiveHeader fuzz, concurrent query on shared reader, concurrent + # reader create+drop lifecycle, streaming benchmark log. + add_paimon_test(tantivy_streaming_test + SOURCES + tantivy_streaming_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-streaming-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # TantivyGlobalIndex + factory + end-to-end integration test. + # `--whole-archive` is required so the static REGISTER_PAIMON_FACTORY + # symbols are not stripped out of the test binary. + add_paimon_test(tantivy_index_test + SOURCES + tantivy_index_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-index-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Cross-implementation coexistence. Links against both the + # lucene and tantivy support static libs to verify they resolve their + # `REGISTER_PAIMON_FACTORY` registrations side by side and don't + # collide on shared symbols. Only built when lucene-fts is enabled. + if(PAIMON_ENABLE_LUCENE) + add_paimon_test(tantivy_lucene_coexist_test + SOURCES + tantivy_lucene_coexist_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-lucene-coexist-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Equivalence + benchmark. Same link line as the coexistence + # test (needs both impls); benchmark output goes to stderr. + add_paimon_test(tantivy_equivalence_test + SOURCES + tantivy_equivalence_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-equivalence-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + endif() +endif() diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp b/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp new file mode 100644 index 00000000..21a009b1 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" + +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/math.h" +#include "paimon/fs/file_system.h" +#include "paimon/io/data_input_stream.h" + +namespace paimon::tantivy { + +namespace { + +/// Wrap the (non-owning) raw InputStream* in a shared_ptr-like handle so +/// DataInputStream — which takes `shared_ptr` — can be used +/// without transferring ownership. We use a no-op deleter to avoid double-free. +struct NoopDeleter { + void operator()(InputStream*) const {} +}; + +} // namespace + +Result ArchiveLayout::Parse(InputStream* in) { + if (in == nullptr) { + return Status::Invalid("ArchiveLayout::Parse: null input stream"); + } + + // DataInputStream defaults to BE — matches paimon-java archive format. + std::shared_ptr wrapped(in, NoopDeleter{}); + DataInputStream dis(wrapped); + + PAIMON_RETURN_NOT_OK(dis.Seek(0)); + + PAIMON_ASSIGN_OR_RAISE(int32_t file_count, dis.ReadValue()); + if (!InRange(file_count)) { + return Status::Invalid(fmt::format("ArchiveLayout::Parse: bad file_count {}", file_count)); + } + + ArchiveLayout layout; + layout.count = static_cast(file_count); + layout.names.reserve(layout.count); + layout.offsets.reserve(layout.count); + layout.lengths.reserve(layout.count); + + for (int32_t i = 0; i < file_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t name_len, dis.ReadValue()); + if (name_len <= 0 || name_len > 1 << 20) { + return Status::Invalid( + fmt::format("ArchiveLayout::Parse: bad name_len {} at entry {}", name_len, i)); + } + std::string name(static_cast(name_len), '\0'); + PAIMON_RETURN_NOT_OK(dis.Read(name.data(), static_cast(name_len))); + + PAIMON_ASSIGN_OR_RAISE(int64_t data_len, dis.ReadValue()); + if (!InRange(data_len)) { + return Status::Invalid( + fmt::format("ArchiveLayout::Parse: bad data_len {} for '{}'", data_len, name)); + } + + PAIMON_ASSIGN_OR_RAISE(int64_t data_offset, dis.GetPos()); + + layout.names.push_back(std::move(name)); + layout.offsets.push_back(static_cast(data_offset)); + layout.lengths.push_back(static_cast(data_len)); + + // Skip past the payload without reading it. + PAIMON_RETURN_NOT_OK(dis.Seek(data_offset + data_len)); + } + + return layout; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.h b/src/paimon/global_index/tantivy/tantivy_archive_layout.h new file mode 100644 index 00000000..73cb3f7f --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_archive_layout.h @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { +class InputStream; +} // namespace paimon + +namespace paimon::tantivy { + +/// Parsed layout of a packed tantivy archive. Arrays are parallel; `count` is +/// their common length. +/// +/// Archive byte format (matches paimon-java `TantivyFullTextGlobalIndexReader. +/// parseArchiveHeader`; big-endian, no version header): +/// `[BE i32 file_count | (BE i32 name_len, name_utf8, BE i64 data_len, data)*]` +/// +/// `offsets[i]` is the archive-absolute byte offset of file `i`'s payload +/// (points past the per-entry header). `lengths[i]` is the payload size. +struct ArchiveLayout { + std::vector names; + std::vector offsets; + std::vector lengths; + std::size_t count = 0; + + /// Read the archive header from `in` (seeking past payloads) and return the + /// layout. Does NOT read file payloads — only header bytes (a few KB). + /// + /// `in` must support `Seek` (all production `paimon::InputStream` subclasses + /// do; we call `Seek(cur + data_len)` to skip over each file's payload). + /// + /// On return, `in`'s internal position is at the end of the archive; callers + /// typically don't care (the stream is subsequently read via pread callbacks). + static Result Parse(InputStream* in); +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_defs.cpp b/src/paimon/global_index/tantivy/tantivy_defs.cpp new file mode 100644 index 00000000..142d727e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_defs.cpp @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_defs.h" + +#include + +namespace paimon::tantivy { + +std::optional GetJiebaDictionaryDirFromEnv() { + const char* env_dir = std::getenv(kJiebaDictDirEnv); + if (env_dir != nullptr && *env_dir != '\0') { + return std::string(env_dir); + } +#ifdef JIEBA_TEST_DICT_DIR + return std::string(JIEBA_TEST_DICT_DIR); +#endif + return std::nullopt; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_defs.h b/src/paimon/global_index/tantivy/tantivy_defs.h new file mode 100644 index 00000000..f9e1ec13 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_defs.h @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon::tantivy { + +/// Identifier used by GlobalIndexFileWriter::NewFileName to prefix on-disk +/// filenames. Tantivy and lucene file prefixes intentionally differ so a +/// reader can dispatch the right implementation by filename pattern. +static inline const char kIdentifier[] = "tantivy-fulltext"; + +/// Schema field names — fixed to match paimon-java. Callers +/// MUST NOT rename these even though `TantivyGlobalIndexWriter::Create` accepts +/// a `field_name` argument (that argument is used only to extract the correct +/// arrow column; the tantivy schema field name is always `"text"`). +static inline const char kTantivyTextFieldName[] = "text"; +static inline const char kTantivyRowIdFieldName[] = "row_id"; + +/// Option-key prefix consumed by TantivyGlobalIndex. Matches the lucene-fts +/// convention so users can configure both implementations with a uniform +/// "." key style. +static inline const char kOptionKeyPrefix[] = "tantivy-fulltext."; + +/// Buffer size for streaming raw packed bytes from FFI to OutputStream +/// (Writer) and from InputStream into Rust (Reader). +static inline const int32_t kDefaultReadBufferSize = 1024 * 1024; +/// Read buffer size knob for the reader. +static inline const char kTantivyReadBufferSize[] = "read.buffer-size"; + +/// If true, omit term frequencies/positions when indexing (smaller index, but +/// no PhraseQuery support). Default false, mirroring lucene-fts. +static inline const char kTantivyWriteOmitTermFreqAndPositions[] = + "write.omit-term-freq-and-position"; + +/// Env var carrying jieba dictionary directory; consumed by both writer and +/// reader. Same name as lucene-fts: a single env var configures both backends. +static inline const char kJiebaDictDirEnv[] = "PAIMON_JIEBA_DICT_DIR"; + +/// Default tokenize mode if not specified in options. +static inline const char kDefaultJiebaTokenizeMode[] = "mix"; +/// Tokenize mode option key. Values: "mp", "mix", "full", "query". +/// "hmm" is rejected with Unsupported (jieba-rs does not expose standalone HMM). +static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode"; + +/// Writer-side tokenizer selector. Values: +/// "default" (default) — tantivy built-in SimpleTokenizer; +/// "paimon_jieba" — jieba-rs CJK tokenizer; opt-in for Chinese workloads +/// "whitespace" / "raw" / "en_stem" — other tantivy built-ins +/// The reader side is schema-driven and auto-dispatches to whatever tokenizer +/// name is baked into the archive, so the default here also determines what +/// paimon-java sees when it cross-reads the archive. +static inline const char kTantivyWriteTokenizer[] = "tantivy.write.tokenizer"; +/// Default tokenizer for writer: tantivy built-in "default" (SimpleTokenizer), +/// chosen so paimon-cpp ↔ paimon-java cross-read works out of the box. +/// Chinese workloads must opt into "paimon_jieba" via kTantivyWriteTokenizer. +static inline const char kDefaultTantivyWriteTokenizer[] = "default"; + +/// Reads the jieba dictionary directory from kJiebaDictDirEnv. Returns the +/// directory when the env var is set and non-empty, otherwise std::nullopt. +/// Shared by the writer and reader so the env-lookup lives in one place; each +/// caller applies its own policy for the missing case (the writer treats it as +/// an error because a jieba index needs a dictionary, while the reader tolerates +/// it because paimon-java archives use the built-in tokenizer and need none). +/// +/// In test builds, falls back to the JIEBA_TEST_DICT_DIR compile-time macro (set +/// on the support objlib) so tests don't have to mutate process-wide env state. +/// Defined in tantivy_defs.cpp (single TU) and mirrors LuceneUtils:: +/// GetJiebaDictionaryDir. +std::optional GetJiebaDictionaryDirFromEnv(); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp new file mode 100644 index 00000000..44f780ac --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Equivalence + benchmark. + * + * EQUIVALENCE: a parametric corpus × query battery that compares lucene-fts + * and tantivy-fulltext result *sets* (doc_id only — not score order, not score + * values). Coverage targets: + * - English bag-of-words: MATCH_ALL / MATCH_ANY / PHRASE / PREFIX / WILDCARD + * - Chinese (jieba "query" mode): MATCH_ALL / MATCH_ANY / PHRASE + * - Pre_filter intersection (no scoring) + * PREFIX and WILDCARD equivalence covers ASCII token patterns, including case + * normalization. Engine-specific regex edge cases are outside this test. + * + * BENCHMARK: build a 200-doc index per backend and time write + 100 queries. + * Prints to stderr; never fails on perf — guarding against perf regressions + * is out of scope here. Numbers are a reportable baseline. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +struct ReaderPair { + std::shared_ptr lucene; + std::shared_ptr tantivy; + std::unique_ptr lucene_root; + std::unique_ptr tantivy_root; +}; + +class TantivyEquivalenceTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + GlobalIndexIOMeta WriteOne(const std::string& factory_id, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array, + const std::string& root) { + EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN( + auto writer, + indexer->CreateWriter("f0", CreateArrowSchema(data_type).get(), file_writer, pool_)); + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_OK(writer->AddBatch(&c_array, std::move(relative_row_ids))); + EXPECT_OK_AND_ASSIGN(auto metas, writer->Finish()); + return metas[0]; + } + + std::shared_ptr OpenOne(const std::string& factory_id, + const std::shared_ptr& data_type, + const std::map& options, + const GlobalIndexIOMeta& meta, + const std::string& root) { + EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(CreateArrowSchema(data_type).get(), + file_reader, {meta}, pool_)); + return reader; + } + + /// Build BOTH lucene + tantivy indexes for the same corpus + options. + /// Returns an opened-reader pair plus owning UniqueTestDirectory handles. + ReaderPair WriteAndOpenBoth(const std::shared_ptr& data_type, + const std::shared_ptr& array, + std::map lucene_opts, + const std::map& tantivy_opts) { + auto lroot = paimon::test::UniqueTestDirectory::Create(); + auto troot = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(lroot && troot); + // lucene requires a tmp directory option; reuse lroot if caller didn't set one. + lucene_opts.emplace("lucene-fts.write.tmp.directory", lroot->Str()); + auto lmeta = WriteOne("lucene-fts", data_type, lucene_opts, array, lroot->Str()); + auto tmeta = WriteOne("tantivy-fulltext", data_type, tantivy_opts, array, troot->Str()); + ReaderPair p; + p.lucene = OpenOne("lucene-fts", data_type, lucene_opts, lmeta, lroot->Str()); + p.tantivy = OpenOne("tantivy-fulltext", data_type, tantivy_opts, tmeta, troot->Str()); + p.lucene_root = std::move(lroot); + p.tantivy_root = std::move(troot); + return p; + } + + static std::set Ids(const std::shared_ptr& result) { + Result br = Status::Invalid("unrecognized result type"); + if (auto scored = std::dynamic_pointer_cast(result)) { + br = scored->GetBitmap(); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + br = plain->GetBitmap(); + } + EXPECT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, std::move(br)); + std::set out; + if (bitmap) { + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.insert(static_cast(*it)); + } + } + return out; + } + + /// Run a single FullTextSearch through both readers, return (lucene, tantivy) + /// doc id sets. + std::pair, std::set> RunPair( + const ReaderPair& p, const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + auto lr = p.lucene->VisitFullTextSearch( + std::make_shared("f0", limit, q, t, filter)); + auto tr = p.tantivy->VisitFullTextSearch( + std::make_shared("f0", limit, q, t, filter)); + EXPECT_OK_AND_ASSIGN(auto lresult, std::move(lr)); + EXPECT_OK_AND_ASSIGN(auto tresult, std::move(tr)); + return {Ids(lresult), Ids(tresult)}; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +} // namespace + +TEST_F(TantivyEquivalenceTest, EnglishBagOfWordsBattery) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta gamma delta"], + ["alpha alpha alpha beta"], + ["beta gamma delta epsilon"], + ["zeta eta theta iota"], + ["alpha gamma epsilon iota"], + ["lone outlier word here"], + ["alpha beta gamma alpha beta"], + ["delta epsilon zeta eta theta"], + ["nothing matches this row"], + ["alpha"] + ])") + .ValueOrDie(); + auto pair = WriteAndOpenBoth(data_type, array, {}, {}); + + struct Case { + std::string query; + FullTextSearch::SearchType type; + std::set expected_ids; + }; + std::vector cases = { + {"alpha", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha beta gamma", FullTextSearch::SearchType::PHRASE, {}}, + {"beta gamma delta", FullTextSearch::SearchType::PHRASE, {}}, + {"delta epsilon", FullTextSearch::SearchType::PHRASE, {}}, + {"ALP", FullTextSearch::SearchType::PREFIX, {0, 1, 4, 6, 9}}, + {"*ALPHA*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}}, + {"*ALP?A*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}}, + }; + for (const auto& c : cases) { + auto [l, t] = RunPair(pair, c.query, c.type); + ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type); + if (!c.expected_ids.empty()) { + ASSERT_EQ(l, c.expected_ids) << "unexpected matches: query=" << c.query + << " type=" << static_cast(c.type); + } + } +} + +TEST_F(TantivyEquivalenceTest, ChineseQueryModeBattery) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ +["智能助手 AI 模块 开发"], +["智能助手 在 Python 开发 中"], +["AI 助手 开发 框架"], +["智能 模块 技术 实现"], +["发展方向 是 智能 助手"] + ])") + .ValueOrDie(); + std::map lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}}; + std::map topts = { + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto pair = WriteAndOpenBoth(data_type, array, lopts, topts); + + struct Case { + std::string query; + FullTextSearch::SearchType type; + }; + // Note: jieba is shared (same dictionary), so tokenization should agree + // for plain Chinese text. Differences (if any) come from the lowercase / + // stopword normalization step — tested with neutral CJK terms below. + std::vector cases = { + {"智能", FullTextSearch::SearchType::MATCH_ALL}, + {"智能 助手", FullTextSearch::SearchType::MATCH_ALL}, + {"模块", FullTextSearch::SearchType::MATCH_ANY}, + {"发展方向", FullTextSearch::SearchType::PHRASE}, + }; + for (const auto& c : cases) { + auto [l, t] = RunPair(pair, c.query, c.type); + ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type); + } +} + +TEST_F(TantivyEquivalenceTest, MixedAsciiCjkPrefixAndWildcard) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["B超检查"], + ["T恤"] + ])") + .ValueOrDie(); + std::map lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}}; + std::map topts = { + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto pair = WriteAndOpenBoth(data_type, array, lopts, topts); + + { + auto [l, t] = RunPair(pair, "B", FullTextSearch::SearchType::PREFIX); + ASSERT_EQ(l, (std::set{0})); + ASSERT_EQ(t, l); + } + { + auto [l, t] = RunPair(pair, "*T*", FullTextSearch::SearchType::WILDCARD); + ASSERT_EQ(l, (std::set{1})); + ASSERT_EQ(t, l); + } +} + +TEST_F(TantivyEquivalenceTest, PreFilterIntersectionEquivalent) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta"], + ["alpha gamma"], + ["alpha delta"], + ["beta gamma"], + ["beta delta"] + ])") + .ValueOrDie(); + auto pair = WriteAndOpenBoth(data_type, array, {}, {}); + + auto pf = RoaringBitmap64::From({0l, 2l, 4l}); + { + auto [l, t] = + RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, pf); + ASSERT_EQ(l, t); + ASSERT_EQ(l, (std::set{0, 2})); + } + { + auto [l, t] = + RunPair(pair, "beta gamma", FullTextSearch::SearchType::MATCH_ANY, std::nullopt, pf); + ASSERT_EQ(l, t); + } + { + auto empty = RoaringBitmap64(); + auto [l, t] = + RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, empty); + ASSERT_EQ(l, t); + ASSERT_TRUE(l.empty()); + } +} + +TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { + // Build a synthetic 200-doc corpus and time write + 100 random queries. + // This is a reportable baseline, NOT a perf gate — assertions only check + // semantic correctness (each query returns >= 0 docs without erroring). + constexpr int32_t kDocCount = 200; + constexpr int32_t kQueryCount = 100; + std::vector vocab = {"alpha", "beta", "gamma", "delta", "epsilon", + "zeta", "eta", "theta", "iota", "kappa", + "lambda", "mu", "nu", "xi", "omicron"}; + std::mt19937 rng(0xC0DE); + std::uniform_int_distribution word_pick(0, vocab.size() - 1); + std::uniform_int_distribution word_count(3, 12); + + // Build the corpus as a JSON Arrow array. + std::string json = "["; + for (int32_t i = 0; i < kDocCount; ++i) { + json += "[\""; + int32_t n = word_count(rng); + for (int32_t w = 0; w < n; ++w) { + if (w > 0) { + json += ' '; + } + json += vocab[word_pick(rng)]; + } + json += "\"]"; + if (i + 1 < kDocCount) { + json += ","; + } + } + json += "]"; + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie(); + + auto time_ms = [](auto&& fn) { + auto t0 = std::chrono::steady_clock::now(); + fn(); + auto t1 = std::chrono::steady_clock::now(); + return std::chrono::duration_cast(t1 - t0).count(); + }; + + // -------- Lucene: write + open + queries -------- + auto lroot = paimon::test::UniqueTestDirectory::Create(); + std::map lopt = {{"lucene-fts.write.tmp.directory", lroot->Str()}}; + GlobalIndexIOMeta lmeta{"", 0, nullptr}; + auto lwrite_ms = + time_ms([&] { lmeta = WriteOne("lucene-fts", data_type, lopt, array, lroot->Str()); }); + auto lreader = OpenOne("lucene-fts", data_type, lopt, lmeta, lroot->Str()); + + auto lquery_ms = time_ms([&] { + for (int32_t i = 0; i < kQueryCount; ++i) { + const std::string& w = vocab[word_pick(rng)]; + ASSERT_OK_AND_ASSIGN( + auto r, + lreader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + } + }); + + // -------- Tantivy: write + open + queries -------- + auto troot = paimon::test::UniqueTestDirectory::Create(); + GlobalIndexIOMeta tmeta{"", 0, nullptr}; + auto twrite_ms = + time_ms([&] { tmeta = WriteOne("tantivy-fulltext", data_type, {}, array, troot->Str()); }); + auto treader = OpenOne("tantivy-fulltext", data_type, {}, tmeta, troot->Str()); + + auto tquery_ms = time_ms([&] { + for (int32_t i = 0; i < kQueryCount; ++i) { + const std::string& w = vocab[word_pick(rng)]; + ASSERT_OK_AND_ASSIGN( + auto r, + treader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + } + }); + + std::cerr << fmt::format( + "[STAGE10-BENCH docs={} queries={}] lucene_write={}ms lucene_query={}ms" + " tantivy_write={}ms tantivy_query={}ms file_size_lucene={} file_size_tantivy={}\n", + kDocCount, kQueryCount, lwrite_ms, lquery_ms, twrite_ms, tquery_ms, lmeta.file_size, + tmeta.file_size); + SUCCEED() << "benchmark prints to stderr"; +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_handle.h b/src/paimon/global_index/tantivy/tantivy_ffi_handle.h new file mode 100644 index 00000000..3c8ecac2 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_handle.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +/// Deleter template; specialize per handle type with the matching free function. +/// Usage: +/// template <> struct FfiDeleter { +/// void operator()(paimon_tantivy_writer_t* p) const noexcept { +/// paimon_tantivy_writer_free(p); +/// } +/// }; +/// using WriterPtr = FfiUniquePtr; +template +struct FfiDeleter { + // Default unsupported so missing specializations fail at compile time + void operator()(Handle*) const noexcept { + static_assert(sizeof(Handle) == 0, "FfiDeleter must be specialized for this handle type"); + } +}; + +/// Generic RAII owning pointer for an FFI handle. +template +using FfiUniquePtr = std::unique_ptr>; + +/// Tokenizer handle. +template <> +struct FfiDeleter { + void operator()(PaimonJiebaTokenizer* p) const noexcept { + paimon_tantivy_tokenizer_free(p); + } +}; +using JiebaTokenizerPtr = FfiUniquePtr; + +/// Writer handle. +template <> +struct FfiDeleter { + void operator()(PaimonTantivyWriter* p) const noexcept { + paimon_tantivy_writer_free(p); + } +}; +using WriterPtr = FfiUniquePtr; + +/// Reader handle. +template <> +struct FfiDeleter { + void operator()(PaimonTantivyReader* p) const noexcept { + paimon_tantivy_reader_free(p); + } +}; +using ReaderPtr = FfiUniquePtr; + +/// Specialization: buffer_t is special - not an opaque handle but a value +/// struct owned on the stack. The contained `data` pointer is the Rust-owned +/// allocation; we call `paimon_tantivy_buffer_free` on the struct pointer. +/// Use BufferGuard to ensure free-on-scope-exit even on early return. +class BufferGuard { + public: + BufferGuard() noexcept { + buf_.data = nullptr; + buf_.len = 0; + buf_.capacity = 0; + } + BufferGuard(const BufferGuard&) = delete; + BufferGuard& operator=(const BufferGuard&) = delete; + BufferGuard(BufferGuard&&) = delete; + BufferGuard& operator=(BufferGuard&&) = delete; + + ~BufferGuard() noexcept { + paimon_tantivy_buffer_free(&buf_); + } + + PaimonTantivyBuffer* out() noexcept { + return &buf_; + } + + const uint8_t* data() const noexcept { + return buf_.data; + } + std::size_t size() const noexcept { + return buf_.len; + } + + private: + PaimonTantivyBuffer buf_{}; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp new file mode 100644 index 00000000..151cc1e9 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" + +#include +#include + +#include "glog/logging.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +// Not in an anonymous namespace: `extern "C"` (external linkage) and an +// anonymous namespace (internal linkage) are contradictory. Only referenced +// via function pointer below, so symbol visibility is not a concern. +/// Level mapping matches Rust side (0=trace..4=error). +extern "C" void PaimonTantivyLogAdapter(int32_t level, const char* msg, std::size_t len) { + // msg is NOT null-terminated; slice with len. + std::string s(msg, len); + switch (level) { + case 4: + LOG(ERROR) << "[tantivy] " << s; + break; + case 3: + LOG(WARNING) << "[tantivy] " << s; + break; + case 2: + LOG(INFO) << "[tantivy] " << s; + break; + case 1: + VLOG(1) << "[tantivy] " << s; + break; + case 0: + VLOG(2) << "[tantivy] " << s; + break; + default: + LOG(INFO) << "[tantivy:lvl=" << level << "] " << s; + break; + } +} + +void InstallTantivyLogBridge() { + paimon_tantivy_set_log_callback(&PaimonTantivyLogAdapter); +} + +void UninstallTantivyLogBridge() { + paimon_tantivy_clear_log_callback(); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.h b/src/paimon/global_index/tantivy/tantivy_ffi_log.h new file mode 100644 index 00000000..7fad3299 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_log.h @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +namespace paimon::tantivy { + +/// Install the Rust -> C++ log callback. Idempotent; only the last caller's +/// callback is active. Threading: C callback runs on tantivy worker threads; +/// our adapter must be thread-safe (it routes to glog which is). +void InstallTantivyLogBridge(); + +/// Uninstall (revert to Rust stderr). Mostly useful for tests. +void UninstallTantivyLogBridge(); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_status.h b/src/paimon/global_index/tantivy/tantivy_ffi_status.h new file mode 100644 index 00000000..74c0fd3e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_status.h @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +/// Translate an FFI status code to a paimon::Status. OK returns Status::OK(). +/// On error, the returned Status carries the thread-local last_error() text +/// prefixed with the status code name for easier grep. +/// +/// Note: cbindgen emits `PaimonTantivyStatus` in the **global** namespace as +/// a C-style enum, so we accept it via its global type here. C++ ADL still +/// lets call sites write the unqualified enumerator names. +inline Status FfiStatusToStatus(::PaimonTantivyStatus code) { + if (code == PAIMON_TANTIVY_STATUS_OK) { + return Status::OK(); + } + const char* err = paimon_tantivy_last_error(); + const char* name = [code]() -> const char* { + switch (code) { + case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT: + return "InvalidArgument"; + case PAIMON_TANTIVY_STATUS_NOT_FOUND: + return "NotFound"; + case PAIMON_TANTIVY_STATUS_IO_ERROR: + return "IoError"; + case PAIMON_TANTIVY_STATUS_UNSUPPORTED: + return "Unsupported"; + case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR: + return "TokenizerError"; + case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR: + return "QueryParseError"; + case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR: + return "IndexFormatError"; + case PAIMON_TANTIVY_STATUS_INTERNAL_ERROR: + return "InternalError"; + default: + return "UnknownFfiStatus"; + } + }(); + std::string msg = fmt::format("tantivy-ffi[{}({})]: {}", name, static_cast(code), + err ? err : "(null)"); + switch (code) { + case PAIMON_TANTIVY_STATUS_NOT_FOUND: + return Status::NotExist(msg); + case PAIMON_TANTIVY_STATUS_IO_ERROR: + return Status::IOError(msg); + case PAIMON_TANTIVY_STATUS_UNSUPPORTED: + return Status::NotImplemented(msg); + case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT: + case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR: + case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR: + case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR: + return Status::Invalid(msg); + default: + return Status::UnknownError(msg); + } +} + +/// Like PAIMON_RETURN_NOT_OK but for FFI calls returning PaimonTantivyStatus. +#define PAIMON_TANTIVY_RETURN_NOT_OK(expr) \ + do { \ + ::PaimonTantivyStatus _paimon_tantivy_status_ = (expr); \ + if (_paimon_tantivy_status_ != PAIMON_TANTIVY_STATUS_OK) { \ + return ::paimon::tantivy::FfiStatusToStatus(_paimon_tantivy_status_); \ + } \ + } while (0) + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp new file mode 100644 index 00000000..a10f3231 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy::test { + +// ------------------------- last_error contract ------------------------- + +TEST(TantivyFfiError, LastErrorIsNeverNull) { + // Before anything, last_error should be a valid non-null pointer to "" + const char* ptr = paimon_tantivy_last_error(); + ASSERT_NE(ptr, nullptr); + // Content is thread-local; for freshly-spawned thread it must be empty + std::atomic child_ok{false}; + std::thread t([&]() { + const char* p = paimon_tantivy_last_error(); + child_ok.store(p != nullptr && p[0] == '\0'); + }); + t.join(); + ASSERT_TRUE(child_ok.load()); +} + +// ------------------------- status translation ------------------------- + +TEST(TantivyFfiStatus, OkTranslates) { + Status s = FfiStatusToStatus(PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK); + ASSERT_TRUE(s.ok()) << s.ToString(); +} + +TEST(TantivyFfiStatus, ErrorCodeNamesShowUp) { + // Translate a few codes and ensure the name appears in the string form. + struct Case { + PaimonTantivyStatus code; + const char* expected_substr; + }; + const Case cases[] = { + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT, "InvalidArgument"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_NOT_FOUND, "NotFound"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_IO_ERROR, "IoError"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_UNSUPPORTED, "Unsupported"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR, "TokenizerError"}, + }; + for (const auto& c : cases) { + Status s = FfiStatusToStatus(c.code); + ASSERT_FALSE(s.ok()); + ASSERT_NE(s.ToString().find(c.expected_substr), std::string::npos) + << "got: " << s.ToString(); + } +} + +// ------------------------- buffer lifetime ------------------------- + +TEST(TantivyFfiBuffer, EmptyBufferGuard) { + BufferGuard g; + ASSERT_EQ(g.size(), 0u); + ASSERT_EQ(g.data(), nullptr); + // Destructor must accept empty buffer +} + +// ------------------------- handle stress ------------------------- + +// Sanity stress: create/destroy a dummy "handle" via into_handle/free_handle. +// Since the Rust side doesn't yet export writer/reader, we stress via a +// temporary wrapping of the buffer API: alloc buffers repeatedly, ensure no +// crash (LSAN / ASAN would catch leaks). +TEST(TantivyFfiBuffer, StressAllocFree) { + for (int32_t i = 0; i < 1000; ++i) { + BufferGuard g; + // We don't have a way to populate the buffer from C++ here; + // this just exercises empty construction + destruction path. + (void)g; + } +} + +// ------------------------- log bridge ------------------------- + +namespace { +std::atomic g_log_count{0}; +extern "C" void CountingLogCb(int32_t /*level*/, const char* /*msg*/, std::size_t /*len*/) { + g_log_count.fetch_add(1, std::memory_order_relaxed); +} +} // namespace + +TEST(TantivyFfiLog, SetCallbackIsIdempotent) { + g_log_count.store(0); + paimon_tantivy_set_log_callback(&CountingLogCb); + paimon_tantivy_set_log_callback(&CountingLogCb); + paimon_tantivy_clear_log_callback(); + // Should not crash even though called multiple times (idempotent install) + SUCCEED(); +} + +TEST(TantivyFfiLog, InstallBridgeThenUninstall) { + // Bridge to glog; must not crash. + InstallTantivyLogBridge(); + UninstallTantivyLogBridge(); + SUCCEED(); +} + +// ------------------------- version still works ------------------------- + +TEST(TantivyFfi, VersionReachable) { + const char* v = paimon_tantivy_version(); + ASSERT_NE(v, nullptr); + ASSERT_GT(std::strlen(v), 0u); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp b/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp new file mode 100644 index 00000000..5de18683 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class TantivyFilterLimitTest : public ::testing::Test { + public: + std::pair, GlobalIndexIOMeta> WriteAndOpen( + const std::shared_ptr& array, + const std::map& options) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(root_dir); + std::string root = root_dir->Str(); + kept_dirs_.push_back(std::move(root_dir)); + auto path_factory = std::make_shared(root); + auto fm = std::make_shared(fs_, path_factory); + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + EXPECT_OK_AND_ASSIGN(auto writer_res, TantivyGlobalIndexWriter::Create( + "f0", data_type, fm, options, GetDefaultPool())); + auto writer = writer_res; + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_TRUE(writer->AddBatch(&c_array, std::move(relative_row_ids)).ok()); + EXPECT_OK_AND_ASSIGN(auto metas_res, writer->Finish()); + return {fm, metas_res[0]}; + } + + static std::vector BitmapToVec(const RoaringBitmap64& b) { + std::vector ids; + for (auto it = b.Begin(); it != b.End(); ++it) { + ids.push_back(static_cast(*it)); + } + std::sort(ids.begin(), ids.end()); + return ids; + } + + std::shared_ptr DataType() const { + return arrow::struct_({arrow::field("f0", arrow::utf8())}); + } + + protected: + std::shared_ptr fs_ = std::make_shared(); + std::vector> kept_dirs_; +}; + +} // namespace + +TEST_F(TantivyFilterLimitTest, LimitProducesScoredResultTopN) { + // Three docs with very different term frequencies for "doc"; limit=2 must + // pick the top 2 by score (doc 1 highest, then doc 2). + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored) << "expected BitmapScoredGlobalIndexResult"; + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + auto ids = BitmapToVec(*bitmap); + ASSERT_EQ(ids, (std::vector{1, 2})); + ASSERT_EQ(scored->GetScores().size(), 2u); + // Per-doc scores must be > 0 and present in iteration (doc-id) order. + for (auto s : scored->GetScores()) { + ASSERT_GT(s, 0.0f); + } +} + +TEST_F(TantivyFilterLimitTest, NoLimitReturnsBitmapResult) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["other"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "doc", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt))); + // No limit ⇒ NOT a BitmapScoredGlobalIndexResult; just BitmapGlobalIndexResult. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1})); +} + +TEST_F(TantivyFilterLimitTest, PreFilterIntersectsWithoutLimit) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["alpha"], ["alpha"], ["alpha"], ["beta"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "alpha", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({0l, 2l, 100l})))); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 2})); +} + +TEST_F(TantivyFilterLimitTest, PreFilterAppliedBeforeLimit) { + // doc 0 has highest score for "doc" but is excluded by pre_filter; the + // result must contain doc 1 only, even with limit=10. + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc doc doc doc doc"], + ["doc doc"], + ["doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/10, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({1l})); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{1})); + ASSERT_EQ(scored->GetScores().size(), 1u); +} + +TEST_F(TantivyFilterLimitTest, EmptyPreFilterReturnsEmpty) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["alpha"], ["beta"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + RoaringBitmap64 empty; // explicitly empty + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "alpha", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/empty))); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_TRUE(bitmap->IsEmpty()); +} + +TEST_F(TantivyFilterLimitTest, LimitGreaterThanMatchesReturnsAll) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["other"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/100, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1})); + ASSERT_EQ(scored->GetScores().size(), 2u); +} + +// =========================================================================== +// v0.2: with_score × limit 4-path matrix guards +// =========================================================================== +// Decouple with_score from limit. The four combinations must each map to the +// correct concrete result type and content. + +// Path A: with_score=false, limit=None → BitmapGlobalIndexResult, all rows, no score. +TEST_F(TantivyFilterLimitTest, WithScoreFalseLimitNoneAllRowsNoScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/std::nullopt, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = false; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // Must NOT be scored. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1, 2})); +} + +// Path B: with_score=false, limit=N → BitmapGlobalIndexResult, any N matches, +// no scoring (no BM25 sort). Used by `WHERE MATCH ... LIMIT N` without ORDER BY. +TEST_F(TantivyFilterLimitTest, WithScoreFalseLimitNAnyNNoScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = false; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // Must NOT be scored. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + // Only cardinality matters — selection order is arbitrary and depends on + // tantivy's posting iteration; the two returned row_ids must each be one + // of the three input docs. + ASSERT_EQ(bitmap->Cardinality(), 2u); + auto vec = BitmapToVec(*bitmap); + for (auto id : vec) { + ASSERT_TRUE(id == 0 || id == 1 || id == 2); + } +} + +// Path C (new in v0.2): with_score=true, limit=None → BitmapScoredGlobalIndexResult, +// all rows + all scores, ordered by row_id asc. +TEST_F(TantivyFilterLimitTest, WithScoreTrueLimitNoneAllRowsWithScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/std::nullopt, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored) << "with_score=true must produce BitmapScoredGlobalIndexResult"; + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1, 2})); + // All 3 docs have scores; sizes must match. + ASSERT_EQ(scored->GetScores().size(), 3u); + for (auto s : scored->GetScores()) { + ASSERT_GT(s, 0.0f); + } +} + +// Path D: with_score=true, limit=N → BitmapScoredGlobalIndexResult, top-N with scores. +// Equivalent to the v0.1 happy-path (LimitProducesScoredResultTopN), kept here +// as an explicit anchor of the 4-path matrix. +TEST_F(TantivyFilterLimitTest, WithScoreTrueLimitNTopNWithScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(bitmap->Cardinality(), 2u); + ASSERT_TRUE(bitmap->Contains(1)); // highest TF must be included + ASSERT_EQ(scored->GetScores().size(), 2u); +} + +// Migration guard: when caller omits `with_score`, the default is `false` — +// even with limit set, the result is a BitmapGlobalIndexResult (NOT scored). +// This catches v0.1 callers that relied on `limit >= 0` to implicitly get scores. +TEST_F(TantivyFilterLimitTest, WithScoreDefaultIsFalse) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + // Note: NOT setting fts->with_score; relying on the default value. + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // v0.2 contract: with_score defaults to false, so even with limit set the + // result is BitmapGlobalIndexResult (NOT BitmapScoredGlobalIndexResult). + ASSERT_FALSE(std::dynamic_pointer_cast(res)) + << "v0.2: limit alone must NOT imply scoring; with_score=true is required"; + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.cpp b/src/paimon/global_index/tantivy/tantivy_global_index.cpp new file mode 100644 index 00000000..f281c035 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index.h" + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" + +namespace paimon::tantivy { + +#define CHECK_NOT_NULL(pointer, error_msg) \ + do { \ + if (!(pointer)) { \ + return Status::Invalid(error_msg); \ + } \ + } while (0) + +TantivyGlobalIndex::TantivyGlobalIndex(const std::map& options) + : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} + +Result>> TantivyGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + +Result> TantivyGlobalIndex::CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_type, + arrow::ImportType(arrow_schema)); + auto struct_type = std::dynamic_pointer_cast(arrow_type); + CHECK_NOT_NULL(struct_type, + "arrow schema must be struct type when create TantivyGlobalIndexWriter"); + auto index_field = struct_type->GetFieldByName(field_name); + CHECK_NOT_NULL( + index_field, + fmt::format("field {} not exist in arrow schema when create TantivyGlobalIndexWriter", + field_name)); + if (index_field->type()->id() != arrow::Type::type::STRING) { + return Status::Invalid("field type must be string"); + } + return TantivyGlobalIndexWriter::Create(field_name, arrow_type, file_writer, options_, pool); +} + +Result> TantivyGlobalIndex::CreateReader( + ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, const std::shared_ptr& pool) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, + arrow::ImportSchema(c_arrow_schema)); + if (files.size() != 1) { + return Status::Invalid(fmt::format( + "tantivy index only has one index file per shard, now num: {}", files.size())); + } + if (arrow_schema->num_fields() != 1) { + return Status::Invalid("TantivyGlobalIndex now only support one field"); + } + auto index_field = arrow_schema->field(0); + if (index_field->type()->id() != arrow::Type::type::STRING) { + return Status::Invalid("field type must be string"); + } + return TantivyGlobalIndexReader::Create(index_field->name(), files[0], file_reader, options_, + pool); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.h b/src/paimon/global_index/tantivy/tantivy_global_index.h new file mode 100644 index 00000000..4b7b944f --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" + +namespace paimon::tantivy { + +/// `GlobalIndexer` implementation backed by tantivy-fulltext. Counterpart to +/// `LuceneGlobalIndex`; the two coexist and are NOT cross-readable. Selection +/// between them happens at the factory layer via the `index_type` identifier. +class TantivyGlobalIndex : public GlobalIndexer { + public: + explicit TantivyGlobalIndex(const std::map& options); + + Result>> GetExtraFieldNames() const override; + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override; + + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, + const std::shared_ptr& pool) const override; + + private: + /// Options after the `tantivy-fulltext.` prefix has been stripped. + std::map options_; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp new file mode 100644 index 00000000..654fa54e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_factory.h" + +#include +#include +#include +#include + +#include "paimon/factories/factory.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" + +namespace paimon::tantivy { + +/// Identifier convention: lucene-fts uses "lucene-fts-global"; we use +/// "tantivy-fulltext-global" so `GlobalIndexerFactory::Get("tantivy-fulltext", ...)` +/// (which appends "-global") routes to us. Keeps both backends discoverable +/// via the same lookup path. +const char TantivyGlobalIndexFactory::IDENTIFIER[] = "tantivy-fulltext-global"; + +Result> TantivyGlobalIndexFactory::Create( + const std::map& options) const { + return std::make_unique(options); +} + +REGISTER_PAIMON_FACTORY(TantivyGlobalIndexFactory); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_factory.h b/src/paimon/global_index/tantivy/tantivy_global_index_factory.h new file mode 100644 index 00000000..e9f9863d --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_factory.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" + +namespace paimon::tantivy { + +/// Factory for creating tantivy-fulltext global indexers. Registered into +/// `FactoryCreator` via `REGISTER_PAIMON_FACTORY` so it is selectable +/// alongside `lucene-fts-global` by passing `index_type = "tantivy-fulltext"` +/// (the suffix `-global` is appended automatically by +/// `GlobalIndexerFactory::Get`). +class TantivyGlobalIndexFactory : public GlobalIndexerFactory { + public: + static const char IDENTIFIER[]; + + const char* Identifier() const override { + return IDENTIFIER; + } + + Result> Create( + const std::map& options) const override; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp new file mode 100644 index 00000000..fd357cb4 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +namespace paimon::tantivy { + +namespace { + +// One-shot install of the Rust log bridge so log::warn! in Rust surfaces via glog. +void EnsureTantivyLogBridge() { + static std::once_flag flag; + std::call_once(flag, [] { InstallTantivyLogBridge(); }); +} + +} // namespace + +Result> TantivyGlobalIndexReader::Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool) { + (void)field_name; // Rust-side knows the field via the schema embedded in meta.json + EnsureTantivyLogBridge(); + + std::map write_options; + if (io_meta.metadata) { + PAIMON_RETURN_NOT_OK(RapidJsonUtil::FromJsonString( + std::string(io_meta.metadata->data(), io_meta.metadata->size()), &write_options)); + } + + PAIMON_ASSIGN_OR_RAISE( + std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, std::string(""))); + if (tokenize_mode.empty()) { + // Reader-side option not set; look at the (possibly empty) write_options blob. + // When write_options is empty (paimon-java-written archive), the value below is + // a placeholder that satisfies FFI validation but is discarded at runtime — + // see the comment block above. Do NOT treat the placeholder as a real default + // for jieba indices; jieba archives written by paimon-cpp always stamp their + // chosen mode into metadata, so the placeholder branch never applies to them. + PAIMON_ASSIGN_OR_RAISE( + tokenize_mode, OptionsUtils::GetValueFromMap(write_options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + } + PAIMON_ASSIGN_OR_RAISE( + bool omit_term_freq_and_positions, + OptionsUtils::GetValueFromMap(write_options, kTantivyWriteOmitTermFreqAndPositions, false)); + + // Tolerate a missing jieba dict dir on the read path: paimon-java archives + // use the built-in "default" tokenizer and need no dictionary, so the Rust + // reader ignores dict_dir for them. Archives that do use jieba still get an + // actionable error from the Rust side when the dictionary path is empty. + std::string dict_dir = GetJiebaDictionaryDirFromEnv().value_or(std::string()); + + // Streaming read path: + // 1) open stream + // 2) ArchiveLayout::Parse — reads only header bytes, seeks past payloads + // 3) wrap stream in StreamCtx (owned by Rust via release callback) + // 4) build PaimonStreamCallbacks → paimon_tantivy_reader_new_streaming + // Archive payloads are read lazily through read_at callbacks as tantivy + // accesses posting lists, meta.json, etc. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stream, + file_reader->GetInputStream(io_meta.file_path)); + PAIMON_ASSIGN_OR_RAISE(ArchiveLayout layout, ArchiveLayout::Parse(stream.get())); + + // Transfer stream ownership to a heap-allocated StreamCtx. Ownership passes + // to Rust as soon as paimon_tantivy_reader_new_streaming is called: Rust + // `paimon_cpp_stream_release(ctx)`s it on any failure and on reader drop, so + // C++ must NOT release it after that call. + auto* stream_ctx = new StreamCtx{std::move(stream), {}}; + PaimonStreamCallbacks callbacks{ + static_cast(stream_ctx), + paimon_cpp_stream_read_at, + paimon_cpp_stream_release, + }; + + // Build C-string array pointing into layout.names (stable during this call). + std::vector name_ptrs; + name_ptrs.reserve(layout.count); + for (const auto& n : layout.names) { + name_ptrs.push_back(n.c_str()); + } + + PaimonTantivyReader* raw = nullptr; + ::PaimonTantivyStatus st = paimon_tantivy_reader_new_streaming( + name_ptrs.data(), layout.offsets.data(), layout.lengths.data(), layout.count, callbacks, + tokenize_mode.c_str(), + /*with_position=*/!omit_term_freq_and_positions, dict_dir.c_str(), &raw); + if (st != PAIMON_TANTIVY_STATUS_OK) { + // Rust already released stream_ctx on the failure path; do not release + // it again here (that would double-free the StreamCtx). + PAIMON_TANTIVY_RETURN_NOT_OK(st); + } + return std::shared_ptr( + new TantivyGlobalIndexReader(ReaderPtr(raw), pool)); +} + +Result> TantivyGlobalIndexReader::VisitFullTextSearch( + const std::shared_ptr& full_text_search) { + if (!full_text_search) { + return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer"); + } + + // Serialize pre_filter (if any) to croaring portable bytes for FFI. + // NB: Serialize() returns a pooled_unique_ptr with MemoryPool::AllocatorDelete; + // converting via raw.release() + shared_ptr(raw_ptr) would substitute + // std::default_delete, causing an alloc/dealloc mismatch (malloc vs operator + // delete). Move directly into shared_ptr so the pooled deleter is preserved + // in the control block. + PAIMON_UNIQUE_PTR pre_filter_bytes_owned; + const char* pre_filter_ptr = nullptr; + std::size_t pre_filter_len = 0; + if (full_text_search->pre_filter.has_value()) { + pre_filter_bytes_owned = full_text_search->pre_filter.value().Serialize(pool_.get()); + pre_filter_ptr = pre_filter_bytes_owned->data(); + pre_filter_len = pre_filter_bytes_owned->size(); + } + + int32_t limit_arg = full_text_search->limit.has_value() + ? static_cast(full_text_search->limit.value()) + : -1; + + float min_score_arg = + full_text_search->min_score.has_value() ? full_text_search->min_score.value() : 0.0f; + + BufferGuard out; + PaimonTantivyStatus st = paimon_tantivy_reader_search( + reader_.get(), static_cast(full_text_search->search_type), + full_text_search->query.data(), full_text_search->query.size(), + full_text_search->with_score, limit_arg, pre_filter_ptr, pre_filter_len, min_score_arg, + out.out()); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + + // Decode `[u8 has_scores | u64 count | u64 row_ids[] | optional f32 scores[]]`. + // row_id is the explicit u64 column read from the fast field. + if (out.size() < 9) { + return Status::Invalid( + fmt::format("tantivy reader output too small ({} bytes)", out.size())); + } + const uint8_t* p = out.data(); + bool has_scores = (p[0] != 0); + // v0.2 consistency check: the wire-level has_scores byte must match the caller's + // with_score flag. A mismatch would indicate FFI / wire-protocol drift. + if (has_scores != full_text_search->with_score) { + return Status::Invalid(fmt::format( + "tantivy wire protocol mismatch: caller with_score={} but buffer has_scores={}", + full_text_search->with_score, has_scores)); + } + uint64_t count; + std::memcpy(&count, p + 1, sizeof(uint64_t)); + std::size_t expected = 1 + 8 + count * 8 + (has_scores ? count * 4 : 0); + if (out.size() != expected) { + return Status::Invalid(fmt::format( + "tantivy reader output size mismatch: has_scores={} count={} expected {} bytes, got {}", + has_scores, count, expected, out.size())); + } + + const uint8_t* row_id_p = p + 9; + if (!has_scores) { + // Copy the contiguous u64 block (unaligned at offset 9) into an aligned + // buffer, then bulk-insert via AddMany instead of a per-row Add loop. + std::vector row_ids(count); + if (count > 0) { + std::memcpy(row_ids.data(), row_id_p, count * sizeof(uint64_t)); + } + RoaringBitmap64 bitmap; + bitmap.AddMany(row_ids.size(), row_ids.data()); + return std::make_shared( + [b = std::move(bitmap)]() -> Result { return b; }); + } + // has_scores=true: produce BitmapScoredGlobalIndexResult. Rust may send rows + // in either row_id-asc order (path C: with_score=true, limit=None) or score-desc + // order (path D: with_score=true, limit=Some). The bitmap iteration order is + // row_id-asc (RoaringBitmap set semantics), so we always re-sort by row_id here + // to keep `scores[i]` aligned with the i-th row_id from the bitmap iterator — + // matching the contract documented in BitmapScoredGlobalIndexResult. + const uint8_t* score_p = row_id_p + count * 8; + std::vector> id_score_pairs; + id_score_pairs.reserve(count); + for (uint64_t i = 0; i < count; i++) { + uint64_t row_id; + std::memcpy(&row_id, row_id_p + i * 8, sizeof(uint64_t)); + float score; + std::memcpy(&score, score_p + i * 4, sizeof(float)); + id_score_pairs.emplace_back(static_cast(row_id), score); + } + // Sort by row_id ascending so scores align with bitmap iteration order. + std::sort(id_score_pairs.begin(), id_score_pairs.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(id_score_pairs.size()); + for (const auto& [id, sc] : id_score_pairs) { + bitmap.Add(id); + scores.push_back(sc); + } + return std::make_shared(std::move(bitmap), std::move(scores)); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_reader.h b/src/paimon/global_index/tantivy/tantivy_global_index_reader.h new file mode 100644 index 00000000..fcfdc455 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_reader.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_search.h" + +namespace paimon::tantivy { + +/// Tantivy-backed implementation of `GlobalIndexReader`. +/// +/// Mirrors LuceneGlobalIndexReader's surface but delegates query construction +/// + execution into Rust over FFI. Supports the 5 FullTextSearch SearchTypes +/// (MATCH_ALL, MATCH_ANY, PHRASE, PREFIX, WILDCARD), optionally with limit and +/// pre_filter. +/// +/// All non-FullTextSearch visit methods return nullptr (matches +/// LuceneGlobalIndexReader): the FTS index has no contribution for non-FTS +/// predicates, framework treats nullptr as "no filter constraint". +class TantivyGlobalIndexReader : public GlobalIndexReader { + public: + static Result> Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool); + + // === FunctionVisitor surface — non-FTS predicates fall back to full range. === + + Result> VisitIsNotNull() override { + return CreateAllResult(); + } + Result> VisitIsNull() override { + return CreateAllResult(); + } + Result> VisitEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitNotEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLessThan(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLessOrEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitGreaterThan(const Literal&) override { + return CreateAllResult(); + } + Result> VisitGreaterOrEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitIn(const std::vector&) override { + return CreateAllResult(); + } + Result> VisitNotIn(const std::vector&) override { + return CreateAllResult(); + } + Result> VisitStartsWith(const Literal&) override { + return CreateAllResult(); + } + Result> VisitEndsWith(const Literal&) override { + return CreateAllResult(); + } + Result> VisitContains(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLike(const Literal&) override { + return CreateAllResult(); + } + + Result> VisitVectorSearch( + const std::shared_ptr&) override { + return Status::Invalid( + "TantivyGlobalIndexReader is not supposed to handle vector search query"); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override; + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return kIdentifier; + } + + private: + TantivyGlobalIndexReader(ReaderPtr reader, std::shared_ptr pool) + : reader_(std::move(reader)), pool_(std::move(pool)) {} + + std::shared_ptr CreateAllResult() const { + return nullptr; + } + + /// Owning handle to the Rust-side reader. + ReaderPtr reader_; + /// MemoryPool used for serializing pre-filter bitmaps to bytes for FFI. + std::shared_ptr pool_; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp new file mode 100644 index 00000000..ea697fa1 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" + +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/global_index/global_index_utils.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +namespace paimon::tantivy { + +#define CHECK_NOT_NULL(pointer, error_msg) \ + do { \ + if (!(pointer)) { \ + return Status::Invalid(error_msg); \ + } \ + } while (0) + +Result> TantivyGlobalIndexWriter::Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + bool omit_term_freq_and_positions, + OptionsUtils::GetValueFromMap(options, kTantivyWriteOmitTermFreqAndPositions, false)); + PAIMON_ASSIGN_OR_RAISE(std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + PAIMON_ASSIGN_OR_RAISE(std::string tokenizer, OptionsUtils::GetValueFromMap( + options, kTantivyWriteTokenizer, + std::string(kDefaultTantivyWriteTokenizer))); + // Jieba dict is only needed when actually using jieba. For tantivy built-in + // tokenizers (e.g. "default") we don't force the caller to ship the jieba + // dict dir — pass an empty string and Rust skips jieba construction. + std::string dict_dir; + if (tokenizer == "paimon_jieba") { + std::optional env_dir = GetJiebaDictionaryDirFromEnv(); + if (!env_dir.has_value()) { + return Status::Invalid(fmt::format( + "jieba dictionary dir not found, please set {} env var", kJiebaDictDirEnv)); + } + dict_dir = std::move(*env_dir); + } + + PaimonTantivyWriter* raw = nullptr; + PaimonTantivyStatus st = paimon_tantivy_writer_new( + field_name.c_str(), tokenize_mode.c_str(), + /*with_position=*/!omit_term_freq_and_positions, dict_dir.c_str(), tokenizer.c_str(), &raw); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + WriterPtr writer(raw); + return std::shared_ptr(new TantivyGlobalIndexWriter( + field_name, arrow_type, std::move(writer), file_writer, options, pool)); +} + +TantivyGlobalIndexWriter::TantivyGlobalIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + WriterPtr writer, const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) + : pool_(pool), + field_name_(field_name), + arrow_type_(arrow_type), + writer_(std::move(writer)), + file_writer_(file_writer), + options_(options) {} + +Status TantivyGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array, + std::vector&& relative_row_ids) { + // First-element check mirrors lucene; trust caller to feed sequential ids + // within a batch (same contract LuceneGlobalIndexWriter relies on). + PAIMON_RETURN_NOT_OK( + GlobalIndexUtils::CheckRelativeRowIds(arrow_array, relative_row_ids, row_id_)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(arrow_array, arrow_type_)); + auto struct_array = std::dynamic_pointer_cast(array); + CHECK_NOT_NULL(struct_array, + "invalid input array in TantivyGlobalIndexWriter, must be struct array"); + auto field_array = struct_array->GetFieldByName(field_name_); + CHECK_NOT_NULL( + field_array, + fmt::format("invalid input array in TantivyGlobalIndexWriter, field {} not in input array", + field_name_)); + auto string_array = std::dynamic_pointer_cast(field_array); + CHECK_NOT_NULL(string_array, + fmt::format("invalid input array in TantivyGlobalIndexWriter, field array {} " + "is not a string array", + field_name_)); + + for (int64_t i = 0; i < string_array->length(); i++) { + const char* text_ptr = nullptr; + size_t text_len = 0; + if (!string_array->IsNull(i)) { + std::string_view view = string_array->Value(i); + text_ptr = view.data(); + text_len = view.size(); + } + // Pass the caller-tracked row_id as an explicit u64 field. + PaimonTantivyStatus st = paimon_tantivy_writer_add( + writer_.get(), static_cast(row_id_), text_ptr, text_len); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + row_id_++; + } + return Status::OK(); +} + +Result> TantivyGlobalIndexWriter::Finish() { + // Streaming finish: open the output file, pipe archive bytes from Rust + // through `paimon_cpp_writer_push` directly into the OutputStream. Peak + // RAM (Rust side) = one fixed streaming buffer, independent of archive size. + PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer_->NewFileName(kIdentifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, + file_writer_->NewOutputStream(index_file_name)); + + WriteCtx ctx{out.get(), Status::OK()}; + PaimonWriteCallbacks cb{ + static_cast(&ctx), + paimon_cpp_writer_push, + }; + + int64_t rust_row_count = 0; + ::PaimonTantivyStatus st = + paimon_tantivy_writer_finish_streaming(writer_.get(), cb, &rust_row_count); + if (st != PAIMON_TANTIVY_STATUS_OK) { + // Prefer the detailed C++-side Status stashed by the write callback + // (if the failure originated there); fall back to FFI-derived status. + if (!ctx.last_error.ok()) { + return ctx.last_error; + } + PAIMON_TANTIVY_RETURN_NOT_OK(st); + } + if (rust_row_count != row_id_) { + return Status::Invalid( + fmt::format("tantivy writer row count {} mismatch paimon inner row count {}", + rust_row_count, row_id_)); + } + + PAIMON_RETURN_NOT_OK(out->Flush()); + PAIMON_RETURN_NOT_OK(out->Close()); + + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name)); + std::string options_json; + PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); + auto meta_bytes = std::make_shared(options_json, pool_.get()); + GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, + /*metadata=*/meta_bytes); + return std::vector({meta}); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.h b/src/paimon/global_index/tantivy/tantivy_global_index_writer.h new file mode 100644 index 00000000..0d3247ab --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.h @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" + +namespace paimon::tantivy { + +/// Tantivy-backed implementation of GlobalIndexWriter. +/// +/// Mirrors LuceneGlobalIndexWriter's lifecycle: +/// Create() → AddBatch()* → Finish() +/// Each shard produces exactly one .index file via the GlobalIndexFileWriter, +/// containing the full packed tantivy on-disk index in a single contiguous blob. +/// +/// Indexes written by this class are NOT cross-readable with lucene-fts. The +/// C++ side of this writer is intentionally thin: index construction, segment +/// merging, and packing all happen in Rust behind the FFI boundary. +class TantivyGlobalIndexWriter : public GlobalIndexWriter { + public: + static Result> Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool); + + ~TantivyGlobalIndexWriter() override = default; + + Status AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) override; + + Result> Finish() override; + + private: + TantivyGlobalIndexWriter(const std::string& field_name, + const std::shared_ptr& arrow_type, WriterPtr writer, + const std::shared_ptr& file_writer, + const std::map& options, + const std::shared_ptr& pool); + + std::shared_ptr pool_; + std::string field_name_; + std::shared_ptr arrow_type_; + /// Owning handle to the Rust-side writer. + WriterPtr writer_; + std::shared_ptr file_writer_; + std::map options_; + /// Last document index processed (matches caller-passed relative_row_ids). + int64_t row_id_ = 0; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_index_test.cpp b/src/paimon/global_index/tantivy/tantivy_index_test.cpp new file mode 100644 index 00000000..199e347b --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_index_test.cpp @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_factory.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class TantivyGlobalIndexIntegrationTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + Result WriteGlobalIndex(const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array, + int64_t /*unused_expected_range_end*/) const { + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr w, + global_index->CreateWriter("f0", CreateArrowSchema(data_type).get(), + file_writer, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + PAIMON_ASSIGN_OR_RAISE(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + auto file_name = PathUtil::GetName(metas[0].file_path); + EXPECT_TRUE(StringUtils::StartsWith(file_name, "tantivy-fulltext-global-index-")) + << file_name; + EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index")); + EXPECT_TRUE(metas[0].metadata); + return metas[0]; + } + + Result> CreateReader( + const std::string& root, const std::shared_ptr& data_type, + const std::map& options, const GlobalIndexIOMeta& meta) const { + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + return global_index->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, + pool_); + } + + void CheckResult(const std::shared_ptr& result, + const std::vector& expected_ids) const { + const RoaringBitmap64* bitmap = nullptr; + if (auto scored = std::dynamic_pointer_cast(result)) { + ASSERT_OK_AND_ASSIGN(bitmap, scored->GetBitmap()); + ASSERT_EQ(scored->GetScores().size(), expected_ids.size()); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + ASSERT_OK_AND_ASSIGN(bitmap, plain->GetBitmap()); + } + ASSERT_TRUE(bitmap); + ASSERT_EQ(*bitmap, RoaringBitmap64::From(expected_ids)) + << "result=" << bitmap->ToString() + << ", expected=" << RoaringBitmap64::From(expected_ids).ToString(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + std::shared_ptr data_type_ = + arrow::struct_({arrow::field("f0", arrow::utf8())}); +}; + +} // namespace + +TEST_F(TantivyGlobalIndexIntegrationTest, EnglishCorpus) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {"tantivy-fulltext.write.omit-term-freq-and-position", "false"}, + }; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["This is an test document."], + ["This is an new document document document."], + ["Document document document document test."], + ["unordered user-defined doc id"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(root, data_type_, options, array, 3)); + ASSERT_EQ(std::string(meta.metadata->data(), meta.metadata->size()), + R"({"write.omit-term-freq-and-position":"false"})"); + + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(root, data_type_, options, meta)); + auto t_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(t_reader); + ASSERT_EQ(t_reader->GetIndexType(), std::string(kIdentifier)); + + auto run = [&](const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + // Use scored path so `limit` returns top-N by BM25, matching test + // expectations (otherwise unscored Path B returns any-N, non-deterministic). + auto fts = std::make_shared("f0", limit, q, t, filter); + fts->with_score = true; + EXPECT_OK_AND_ASSIGN(auto res, t_reader->VisitFullTextSearch(fts)); + return res; + }; + + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10), {2, 1, 0}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ANY, 1), {2}); + CheckResult(run("test document", FullTextSearch::SearchType::MATCH_ALL, 10), {2, 0}); + CheckResult(run("test new", FullTextSearch::SearchType::MATCH_ANY, 10), {1, 0, 2}); + CheckResult(run("test document", FullTextSearch::SearchType::PHRASE, 10), {0}); + CheckResult(run("unordered", FullTextSearch::SearchType::MATCH_ALL, 10), {3}); + CheckResult(run("unorder", FullTextSearch::SearchType::PREFIX, 10), {3}); + CheckResult(run("*order*", FullTextSearch::SearchType::WILDCARD, 10), {3}); + CheckResult(run("*or*er*", FullTextSearch::SearchType::WILDCARD, 10), {3}); + + // pre_filter + CheckResult( + run("document", FullTextSearch::SearchType::MATCH_ALL, 10, RoaringBitmap64::From({0l, 1l})), + {0, 1}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10, + RoaringBitmap64::From({2l, 100l})), + {2}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10, + RoaringBitmap64::From({20l, 100l})), + {}); + + // No limit + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL), {0, 1, 2}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, + RoaringBitmap64::From({2l})), + {2}); + CheckResult(run("document test", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, + RoaringBitmap64::From({1l, 2l, 3l, 100l})), + {2}); + + // Unscored path: no with_score ⇒ BitmapGlobalIndexResult (not scored), same + // matches across all 5 SearchTypes. (Previously covered by reader_test.) + auto run_unscored = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto res, + t_reader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, q, t, std::nullopt))); + EXPECT_FALSE(std::dynamic_pointer_cast(res)) + << "unscored query must not return a scored result"; + return res; + }; + CheckResult(run_unscored("document", FullTextSearch::SearchType::MATCH_ALL), {0, 1, 2}); + CheckResult(run_unscored("test new", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2}); + CheckResult(run_unscored("test document", FullTextSearch::SearchType::PHRASE), {0}); + CheckResult(run_unscored("unorder", FullTextSearch::SearchType::PREFIX), {3}); + CheckResult(run_unscored("*order*", FullTextSearch::SearchType::WILDCARD), {3}); +} + +TEST_F(TantivyGlobalIndexIntegrationTest, ChineseCorpus) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {"tantivy-fulltext.write.omit-term-freq-and-position", "false"}, + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ +["QianWen 是一个基于 AI 的智能助手，类似于 Siri 和 Alexa。我们正在用 Python 开发 QianWen 的 Natural Language Understanding 模块，该模块支持多轮对话和意图识别功能，是新一代智能助手的核心技术之一。"], +["最近开源了一个新项目叫ｑｉａｎｗｅｎ（全角字符），功能类似之前的 Qianwen，是一个面向 AI 应用的智能助手。它不仅支持 Machine Learning 和 NLP 技术，还提供了可扩展的开发框架，便于开发者构建自己的智能助手系统。"], +["我们在测试 qianwen-core v1.2 和 ai-engine-alpha 中的 bug，重点优化了 qianwen 的响应速度和稳定性。本次更新增强了核心模块的功能，提升了智能助手的开发效率，并修复了与 NLP 模块相关的多个问题。"], +["AI 助手开发中常用的技术包括 Speech Recognition、Natural Language Processing 和 Recommendation System。我们使用 TensorFlow 和 PyTorch 构建模型，开发了多个智能助手原型，支持语音交互和上下文理解功能，是当前热门的人工智能发展应用方向。"], +["新一代的 AI 助手代号为「千问」，内部命名为 QianwenX-2024，计划在 next quarter 发布。QianwenX 将集成更强的 multimodel 能力，支持图像和文本联合处理，进一步提升智能助手的理解能力和交互体验，是未来智能助手的重要发展方向。"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(root, data_type_, options, array, 4)); + ASSERT_EQ( + std::string(meta.metadata->data(), meta.metadata->size()), + R"({"jieba.tokenize-mode":"query","tantivy.write.tokenizer":"paimon_jieba","write.omit-term-freq-and-position":"false"})"); + + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(root, data_type_, options, meta)); + auto t_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(t_reader); + + auto run = [&](const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + // Use scored path so `limit` returns top-N by BM25, matching test + // expectations (otherwise unscored Path B returns any-N, non-deterministic). + auto fts = std::make_shared("f0", limit, q, t, filter); + fts->with_score = true; + EXPECT_OK_AND_ASSIGN(auto res, t_reader->VisitFullTextSearch(fts)); + return res; + }; + + CheckResult(run("模块", FullTextSearch::SearchType::MATCH_ALL, 10), {0, 2}); + CheckResult(run("模块", FullTextSearch::SearchType::MATCH_ANY, 1), {0}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ALL, 10), {0}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY, 10), {0, 1, 2, 3}); + CheckResult(run("发展方向", FullTextSearch::SearchType::PHRASE, 10), {4}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY, 10, + RoaringBitmap64::From({1l, 3l, 4l})), + {1, 3}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2, 3}); + + // Unscored path on jieba-tokenized Chinese. (Previously covered by reader_test.) + auto run_unscored = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto res, + t_reader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, q, t, std::nullopt))); + return res; + }; + CheckResult(run_unscored("模块", FullTextSearch::SearchType::MATCH_ALL), {0, 2}); + CheckResult(run_unscored("模块技术", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2, 3}); + CheckResult(run_unscored("发展方向", FullTextSearch::SearchType::PHRASE), {4}); +} + +TEST_F(TantivyGlobalIndexIntegrationTest, FactoryLookupReturnsTantivyIndexer) { + std::map options = { + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + // Identifier passed to GlobalIndexerFactory::Get is the prefix; "-global" + // is appended automatically. So "tantivy-fulltext" must route to our factory. + ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, + GlobalIndexerFactory::Get("tantivy-fulltext", options)); + ASSERT_TRUE(indexer); + auto* casted = dynamic_cast(indexer.get()); + ASSERT_TRUE(casted) << "factory did not return a TantivyGlobalIndex"; +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp new file mode 100644 index 00000000..bd9339e3 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif +#ifndef PAIMON_TANTIVY_JAVA_FIXTURE_DIR +#error "PAIMON_TANTIVY_JAVA_FIXTURE_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FixturePathFactory : public IndexPathFactory { + public: + explicit FixturePathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class JavaCompatTest : public ::testing::Test { + public: + /// Build a TantivyGlobalIndexReader on top of the Java-produced fixture. + /// `fixture_name` is relative to `PAIMON_TANTIVY_JAVA_FIXTURE_DIR`. + std::shared_ptr OpenFixture(const std::string& fixture_name) { + std::string fixture_dir = PAIMON_TANTIVY_JAVA_FIXTURE_DIR; + std::string archive_path = PathUtil::JoinPath(fixture_dir, fixture_name); + + EXPECT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); + int64_t file_size = file_status->GetLen(); + EXPECT_GT(file_size, 4) << "fixture archive must exist and be > 4 bytes"; + + // Empty metadata (options not needed for cross-read — we use defaults) + std::string metadata_json = "{}"; + auto meta_bytes = std::make_shared(metadata_json, pool_.get()); + + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + + std::map options; + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(fixture_dir); + auto file_reader = std::make_shared(fs_, path_factory); + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + + EXPECT_OK_AND_ASSIGN(auto reader_res, global_index->CreateReader( + c_schema.get(), file_reader, {io_meta}, pool_)); + return reader_res; + } + + std::shared_ptr BuildFts(FullTextSearch::SearchType type, + const std::string& query) { + return std::make_shared( + /*_field_name=*/"f0", + /*_limit=*/std::optional{}, + /*_query=*/query, + /*_search_type=*/type, + /*_pre_filter=*/std::optional{}); + } + + /// Run the search and return the sorted row_ids from the result bitmap. + std::vector RunSearchRowIds(const std::shared_ptr& reader, + FullTextSearch::SearchType type, + const std::string& query) { + auto fts = BuildFts(type, query); + // This helper returns a value, so gtest ASSERT_* (which `return;`) cannot + // be used here; use the EXPECT_OK family. + EXPECT_OK_AND_ASSIGN(std::shared_ptr r, + reader->VisitFullTextSearch(fts)); + + const RoaringBitmap64* bitmap = nullptr; + if (auto plain = std::dynamic_pointer_cast(r)) { + EXPECT_OK_AND_ASSIGN(bitmap, plain->GetBitmap()); + } else if (auto scored = std::dynamic_pointer_cast(r)) { + EXPECT_OK_AND_ASSIGN(bitmap, scored->GetBitmap()); + } + EXPECT_TRUE(bitmap != nullptr); + if (bitmap == nullptr) { + return {}; + } + + std::vector out; + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.push_back(static_cast(*it)); + } + std::sort(out.begin(), out.end()); + return out; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +} // namespace + +// ============================================================================ +// 1. Archive basics: opening the Java-produced fixture succeeds +// ============================================================================ + +TEST_F(JavaCompatTest, OpenJavaArchiveSucceeds) { + auto reader = OpenFixture("english_simple.archive"); + ASSERT_TRUE(reader != nullptr); +} + +// ============================================================================ +// 2. MATCH_ALL — single and multi-term +// ============================================================================ + +TEST_F(JavaCompatTest, MatchAllApple) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple"); + // Docs containing "apple": 0 ("apple banana cherry"), 1 ("apple durian"), + // 4 ("apple cherry fig"), 7 ("apple") + ASSERT_EQ(ids, (std::vector{0, 1, 4, 7})); +} + +TEST_F(JavaCompatTest, MatchAllAppleBananaIntersection) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple banana"); + // Only doc 0 contains both "apple" and "banana" + ASSERT_EQ(ids, (std::vector{0})); +} + +// ============================================================================ +// 3. MATCH_ANY — union +// ============================================================================ + +TEST_F(JavaCompatTest, MatchAnyDurianElderberryUnion) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, "durian elderberry"); + // durian: 1, 6 elderberry: 5, 8 union: {1, 5, 6, 8} + ASSERT_EQ(ids, (std::vector{1, 5, 6, 8})); +} + +// ============================================================================ +// 4. PHRASE — consecutive term order matters +// ============================================================================ + +TEST_F(JavaCompatTest, PhraseAppleBanana) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "apple banana"); + // Only doc 0 has "apple banana" as consecutive phrase + ASSERT_EQ(ids, (std::vector{0})); +} + +TEST_F(JavaCompatTest, PhraseBananaCherry) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "banana cherry"); + // "banana cherry" consecutive in doc 0 ("apple banana cherry") and doc 2 ("banana cherry") + ASSERT_EQ(ids, (std::vector{0, 2})); +} + +// ============================================================================ +// 5. PREFIX — byte-level (not tokenized) via RegexQuery +// ============================================================================ + +TEST_F(JavaCompatTest, PrefixAp) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PREFIX, "ap"); + // Tokens starting with "ap": "apple" → docs 0, 1, 4, 7 + ASSERT_EQ(ids, (std::vector{0, 1, 4, 7})); +} + +// ============================================================================ +// 6. WILDCARD — glob-style via regex +// ============================================================================ + +TEST_F(JavaCompatTest, WildcardErr) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::WILDCARD, "*err*"); + // Tokens matching *err*: "cherry" (0,2,4,6,9), "elderberry" (5,8) + ASSERT_EQ(ids, (std::vector{0, 2, 4, 5, 6, 8, 9})); +} + +// ============================================================================ +// 7. row_id invariant — must return the *caller-supplied* row_ids (not doc_ids) +// ============================================================================ + +TEST_F(JavaCompatTest, AllDocsReachableByRowId) { + auto reader = OpenFixture("english_simple.archive"); + // Union of all terms matches all 10 docs. + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, + "apple banana cherry durian fig grape elderberry"); + ASSERT_EQ(ids, (std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + // This confirms Java wrote row_ids 0..9 via `addDocument(rowId, text)` and + // paimon-cpp reader extracted them via fast_fields().u64("row_id") — + // the schema invariant survives round-trip across implementations. +} + +// ============================================================================ +// 8. Probe: real paimon-java production archive (handed over by Java team). +// Data was claimed to be (id INT, content STRING) with 5 rows but ids +// rewritten multiple times; dump layout + per-term hits so caller can +// reverse-engineer what's actually inside. +// ============================================================================ + +TEST_F(JavaCompatTest, ProductionSampleProbe) { + const std::string fixture_name = "production_sample.archive"; + + // Open a reader over the Java-written production sample archive. + auto reader = OpenFixture(fixture_name); + ASSERT_TRUE(reader != nullptr); + + // Keywords expected from the production text samples; tokenizer is "default" + // (lowercased, word-granular). + const std::vector probes = { + "apache", "paimon", "is", "a", "lake", "format", "supports", + "full", "text", "search", "in", "vector", "similarity", "using", + "lumina", "streaming", "and", "batch", "processing", "engine", + }; + + // The archive must be readable — at least one probe term hits. + bool any_hit = false; + for (const auto& term : probes) { + if (!RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, term).empty()) { + any_hit = true; + break; + } + } + ASSERT_TRUE(any_hit) << "no probe term hit; archive may be empty or schema mismatched"; +} + +// ============================================================================ +// 9. Reverse direction: paimon-cpp writes with tokenizer="default" → fixture +// consumed by paimon-java test. This test emits the archive into +// test/test_data/cpp_tantivy_fixtures/english_default.archive and +// round-trips it through the cpp reader first (schema-driven tokenizer +// dispatch picks "default" automatically). +// ============================================================================ + +namespace { + +/// GlobalIndexFileWriter that emits to a single fixed filename under `root`. +/// Mirrors paimon-java's `FixedNameLocalFileWriter` from +/// `TantivyIndexFixtureGen.java`: `newFileName(prefix)` ignores the prefix and +/// always returns the caller-chosen name. Used to produce a stable fixture +/// path consumed by the paimon-java cross-read test. +class FixedNameGlobalIndexFileWriter : public GlobalIndexFileWriter { + public: + FixedNameGlobalIndexFileWriter(std::shared_ptr fs, std::string root, + std::string fixed_name) + : fs_(std::move(fs)), root_(std::move(root)), fixed_name_(std::move(fixed_name)) {} + + Result NewFileName(const std::string& /*prefix*/) const override { + return fixed_name_; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + Result> NewOutputStream( + const std::string& file_name) const override { + return fs_->Create(ToPath(file_name), /*overwrite=*/true); + } + Result GetFileSize(const std::string& file_name) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, + fs_->GetFileStatus(ToPath(file_name))); + return file_status->GetLen(); + } + + private: + std::shared_ptr fs_; + std::string root_; + std::string fixed_name_; +}; + +/// Same 10-doc English corpus paimon-java uses in TantivyIndexFixtureGen +/// (pure ASCII, no punctuation inside words). SimpleTokenizer (tantivy's +/// "default") tokenizes identically on both sides for this subset, so the +/// golden row_ids match byte-for-byte between cpp-write and java-read. +constexpr const char* kEnglishDocs[] = { + "apple banana cherry", // 0 + "apple durian", // 1 + "banana cherry", // 2 + "fig grape", // 3 + "apple cherry fig", // 4 + "banana elderberry", // 5 + "cherry durian", // 6 + "apple", // 7 + "grape fig elderberry", // 8 + "cherry fig", // 9 +}; + +} // namespace + +TEST_F(JavaCompatTest, CppWriteDefaultTokenizerForJavaCrossRead) { + // 1) Produce an archive into test/test_data/cpp_tantivy_fixtures/ via the + // production TantivyGlobalIndexWriter, configured with tantivy's + // built-in "default" tokenizer (same as paimon-java's TEXT field). + const std::string out_dir = PAIMON_TANTIVY_CPP_FIXTURE_DIR; + const std::string fixture_name = "english_default.archive"; + // Ensure dir exists (CMake does NOT create it automatically). + ASSERT_OK(fs_->Mkdirs(out_dir)); + // Clean any prior fixture so each test run writes fresh bytes. + { + const std::string archive_path_cleanup = PathUtil::JoinPath(out_dir, fixture_name); + auto existing = fs_->GetFileStatus(archive_path_cleanup); + if (existing.ok()) { + ASSERT_TRUE(fs_->Delete(archive_path_cleanup, false).ok()); + } + } + + auto file_writer = std::make_shared(fs_, out_dir, fixture_name); + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + + std::map options{ + {kTantivyWriteTokenizer, "default"}, + }; + ASSERT_OK_AND_ASSIGN(auto writer_res, TantivyGlobalIndexWriter::Create( + "f0", data_type, file_writer, options, pool_)); + auto writer = writer_res; + + // Build an arrow batch from kEnglishDocs. + std::string json = "["; + for (std::size_t i = 0; i < sizeof(kEnglishDocs) / sizeof(kEnglishDocs[0]); ++i) { + if (i > 0) { + json += ","; + } + json += "[\""; + json += kEnglishDocs[i]; + json += "\"]"; + } + json += "]"; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie(); + ::ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + ASSERT_TRUE(writer->AddBatch(&c_array, std::move(relative_row_ids)).ok()); + ASSERT_OK_AND_ASSIGN(auto metas_res, writer->Finish()); + ASSERT_EQ(metas_res.size(), 1u); + const auto& meta = metas_res.front(); + const std::string archive_path = meta.file_path; + + // 2) Archive header sanity: 16+ files, meta.json present, tokenizer in schema. + ASSERT_OK_AND_ASSIGN(std::shared_ptr stream, fs_->Open(archive_path)); + ASSERT_OK_AND_ASSIGN(auto layout_res, ArchiveLayout::Parse(stream.get())); + const auto& layout = layout_res; + bool has_meta_json = false; + for (std::size_t i = 0; i < layout.count; ++i) { + if (layout.names[i] == "meta.json") { + has_meta_json = true; + } + } + ASSERT_TRUE(has_meta_json); + + // 3) Round-trip through the cpp reader first — the reader must auto-register + // "default" from the schema so the search path works without passing + // any reader-side tokenizer config. + // Build a reader directly off the archive path (mirrors OpenFixture + // but rooted at the cpp fixtures dir). + ASSERT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); + int64_t file_size = file_status->GetLen(); + auto meta_bytes = std::make_shared(std::string("{}"), pool_.get()); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + auto reader_factory = + std::make_shared(std::map{}); + auto reader_path_factory = std::make_shared(out_dir); + auto reader_file_mgr = std::make_shared(fs_, reader_path_factory); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + + ASSERT_OK_AND_ASSIGN(auto reader_res, reader_factory->CreateReader( + c_schema.get(), reader_file_mgr, {io_meta}, pool_)); + auto reader = reader_res; + + // Golden expectations (identical to paimon-java's english_simple.golden.json) + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple"), + (std::vector{0, 1, 4, 7})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple banana"), + (std::vector{0})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, "durian elderberry"), + (std::vector{1, 5, 6, 8})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "apple banana"), + (std::vector{0})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "banana cherry"), + (std::vector{0, 2})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PREFIX, "ap"), + (std::vector{0, 1, 4, 7})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::WILDCARD, "*err*"), + (std::vector{0, 2, 4, 5, 6, 8, 9})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, + "apple banana cherry durian fig grape elderberry"), + (std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp b/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp new file mode 100644 index 00000000..2b6fdeb5 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Adopt one of the two factory identifiers; everything else (paths, queries, +/// arrow plumbing) is shared. +struct ImplSpec { + std::string factory_id; // "lucene-fts" or "tantivy-fulltext" + std::string file_prefix; // "lucene-fts-global-index-" or "tantivy-fulltext-global-index-" + std::string option_prefix; // "lucene-fts." or "tantivy-fulltext." +}; + +class TantivyLuceneCoexistTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + Result WriteWith(const ImplSpec& impl, const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(impl.factory_id, options)); + if (!indexer) { + return Status::Invalid(fmt::format("factory returned null for {}", impl.factory_id)); + } + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr w, + indexer->CreateWriter("f0", CreateArrowSchema(data_type).get(), file_writer, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + PAIMON_ASSIGN_OR_RAISE(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + EXPECT_TRUE( + StringUtils::StartsWith(PathUtil::GetName(metas[0].file_path), impl.file_prefix)) + << metas[0].file_path << " did not start with " << impl.file_prefix; + return metas[0]; + } + + Result> OpenReader( + const ImplSpec& impl, const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, const GlobalIndexIOMeta& meta) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(impl.factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + return indexer->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, + pool_); + } + + static std::set ExtractDocIds(const std::shared_ptr& result) { + Result br = Status::Invalid("no result"); + if (auto scored = std::dynamic_pointer_cast(result)) { + br = scored->GetBitmap(); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + br = plain->GetBitmap(); + } + EXPECT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, std::move(br)); + std::set out; + if (bitmap) { + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.insert(static_cast(*it)); + } + } + return out; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + + inline static const ImplSpec kLucene{"lucene-fts", "lucene-fts-global-index-", "lucene-fts."}; + inline static const ImplSpec kTantivy{"tantivy-fulltext", "tantivy-fulltext-global-index-", + "tantivy-fulltext."}; +}; + +} // namespace + +TEST_F(TantivyLuceneCoexistTest, BothFactoriesResolve) { + // No options needed; just verify both factories register and dispatch. + ASSERT_OK_AND_ASSIGN(auto lucene_indexer, GlobalIndexerFactory::Get("lucene-fts", {})); + ASSERT_OK_AND_ASSIGN(auto tantivy_indexer, GlobalIndexerFactory::Get("tantivy-fulltext", {})); + ASSERT_TRUE(lucene_indexer); + ASSERT_TRUE(tantivy_indexer); + // Sanity: factories return distinct types — different vtables → different + // GetIndexType() once we open a reader (not testable here without an + // index), so just check shared_ptr identity differs. + ASSERT_NE(static_cast(lucene_indexer.get()), static_cast(tantivy_indexer.get())); +} + +TEST_F(TantivyLuceneCoexistTest, SideBySideEnglishCorpusReturnsSameDocIds) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta gamma document"], + ["alpha alpha document"], + ["gamma delta epsilon"], + ["alpha beta document document"] + ])") + .ValueOrDie(); + + auto lucene_root = paimon::test::UniqueTestDirectory::Create(); + auto tantivy_root = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(lucene_root && tantivy_root); + + // Lucene requires a tmp directory option; tantivy ignores unknown keys. + std::map lucene_options = { + {"lucene-fts.write.tmp.directory", lucene_root->Str()}}; + + // Write through BOTH factories side by side in the same process. + ASSERT_OK_AND_ASSIGN(auto lucene_meta, + WriteWith(kLucene, lucene_root->Str(), data_type, lucene_options, array)); + ASSERT_OK_AND_ASSIGN(auto tantivy_meta, + WriteWith(kTantivy, tantivy_root->Str(), data_type, {}, array)); + + ASSERT_OK_AND_ASSIGN(auto lucene_reader, + OpenReader(kLucene, lucene_root->Str(), data_type, {}, lucene_meta)); + ASSERT_OK_AND_ASSIGN(auto tantivy_reader, + OpenReader(kTantivy, tantivy_root->Str(), data_type, {}, tantivy_meta)); + ASSERT_EQ(lucene_reader->GetIndexType(), std::string("lucene-fts")); + ASSERT_EQ(tantivy_reader->GetIndexType(), std::string("tantivy-fulltext")); + + auto run_pair = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto lr, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, q, t, /*pre_filter=*/std::nullopt))); + EXPECT_OK_AND_ASSIGN(auto tr, + tantivy_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, q, t, /*pre_filter=*/std::nullopt))); + return std::make_pair(ExtractDocIds(lr), ExtractDocIds(tr)); + }; + + // For an English bag-of-words corpus the two implementations should agree + // on which docs contain which terms — Lucene and tantivy both store + // lowercased word tokens. + { + auto [l, t] = run_pair("document", FullTextSearch::SearchType::MATCH_ALL); + ASSERT_EQ(l, t) << "MATCH_ALL document — lucene vs tantivy doc id set differs"; + ASSERT_EQ(l, (std::set{0, 1, 3})); + } + { + auto [l, t] = run_pair("alpha beta", FullTextSearch::SearchType::MATCH_ALL); + ASSERT_EQ(l, t) << "MATCH_ALL 'alpha beta' — sets differ"; + ASSERT_EQ(l, (std::set{0, 3})); + } + { + auto [l, t] = run_pair("alpha epsilon", FullTextSearch::SearchType::MATCH_ANY); + ASSERT_EQ(l, t) << "MATCH_ANY 'alpha epsilon' — sets differ"; + ASSERT_EQ(l, (std::set{0, 1, 2, 3})); + } + { + auto [l, t] = run_pair("alpha beta", FullTextSearch::SearchType::PHRASE); + ASSERT_EQ(l, t) << "PHRASE 'alpha beta' — sets differ"; + ASSERT_EQ(l, (std::set{0, 3})); + } +} + +TEST_F(TantivyLuceneCoexistTest, IndependentLifecycleNoStateLeakage) { + // Build a lucene index and a tantivy index back-to-back many times in the + // same process; if either factory leaked global state across instances + // we'd see crashes or stale results. + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + + for (int32_t round = 0; round < 3; ++round) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["round payload one"], + ["round payload two"] + ])") + .ValueOrDie(); + auto lroot = paimon::test::UniqueTestDirectory::Create(); + auto troot = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(lroot && troot); + + std::map lopt = { + {"lucene-fts.write.tmp.directory", lroot->Str()}}; + ASSERT_OK_AND_ASSIGN(auto lm, WriteWith(kLucene, lroot->Str(), data_type, lopt, array)); + ASSERT_OK_AND_ASSIGN(auto tm, WriteWith(kTantivy, troot->Str(), data_type, {}, array)); + ASSERT_OK_AND_ASSIGN(auto lr, OpenReader(kLucene, lroot->Str(), data_type, {}, lm)); + ASSERT_OK_AND_ASSIGN(auto tr, OpenReader(kTantivy, troot->Str(), data_type, {}, tm)); + + ASSERT_OK_AND_ASSIGN(auto lq, lr->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, "payload", + FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + ASSERT_OK_AND_ASSIGN(auto tq, tr->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, "payload", + FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + ASSERT_EQ(ExtractDocIds(lq), (std::set{0, 1})) << "lucene round " << round; + ASSERT_EQ(ExtractDocIds(tq), (std::set{0, 1})) << "tantivy round " << round; + } +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp b/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp new file mode 100644 index 00000000..9fb17822 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "gtest/gtest.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy::test { + +TEST(TantivySmoke, VersionIsReachable) { + const char* version = paimon_tantivy_version(); + ASSERT_NE(version, nullptr) << "paimon_tantivy_version returned null"; + + const std::string v(version); + ASSERT_FALSE(v.empty()); + // build.rs pins version from Cargo.toml (CARGO_PKG_VERSION), semver "x.y.z" + ASSERT_NE(v.find('.'), std::string::npos) << "expected semver, got: " << v; +} + +TEST(TantivySmoke, VersionPointerIsStable) { + // The pointer is documented as 'static — two calls should return either + // the same pointer or at least equivalent string content. + const char* v1 = paimon_tantivy_version(); + const char* v2 = paimon_tantivy_version(); + ASSERT_NE(v1, nullptr); + ASSERT_NE(v2, nullptr); + ASSERT_EQ(std::strcmp(v1, v2), 0); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp b/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp new file mode 100644 index 00000000..fa12b3de --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +#include + +#include "fmt/format.h" +#include "paimon/fs/file_system.h" + +namespace paimon::tantivy { + +extern "C" int32_t paimon_cpp_stream_read_at(void* ctx_ptr, uint64_t offset, std::size_t len, + uint8_t* out_buf) { + if (ctx_ptr == nullptr || out_buf == nullptr) { + return 1; + } + auto* ctx = static_cast(ctx_ptr); + std::lock_guard lock(ctx->pread_mu); + + std::size_t total = 0; + while (total < len) { + auto r = ctx->stream->Read(reinterpret_cast(out_buf + total), + static_cast(len - total), offset + total); + if (!r.ok()) { + return 1; + } + int32_t got = r.value(); + if (got <= 0) { + return 1; // unexpected EOF / 0-byte read + } + total += static_cast(got); + } + return 0; +} + +extern "C" void paimon_cpp_stream_release(void* ctx_ptr) { + if (ctx_ptr == nullptr) { + return; + } + auto* ctx = static_cast(ctx_ptr); + // ~shared_ptr closes the underlying stream. + delete ctx; +} + +extern "C" int32_t paimon_cpp_writer_push(void* ctx_ptr, const uint8_t* data, std::size_t len) { + if (ctx_ptr == nullptr) { + return 1; + } + auto* ctx = static_cast(ctx_ptr); + if (ctx->out == nullptr) { + ctx->last_error = Status::Invalid("writer_push: null OutputStream"); + return 1; + } + std::size_t total = 0; + while (total < len) { + auto r = ctx->out->Write(reinterpret_cast(data + total), + static_cast(len - total)); + if (!r.ok()) { + ctx->last_error = r.status(); + return 1; + } + int32_t written = r.value(); + if (written <= 0) { + ctx->last_error = Status::IOError(fmt::format( + "writer_push: short write (wrote {} of {} bytes)", written, len - total)); + return 1; + } + total += static_cast(written); + } + return 0; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_stream_ctx.h b/src/paimon/global_index/tantivy/tantivy_stream_ctx.h new file mode 100644 index 00000000..a4cb02f7 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_stream_ctx.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/status.h" + +namespace paimon { +class InputStream; +class OutputStream; +} // namespace paimon + +namespace paimon::tantivy { + +/// C++ side wrapper around a seekable InputStream, used as the `ctx` of +/// `PaimonStreamCallbacks`. Lifetime is transferred to Rust via +/// `paimon_tantivy_reader_new_streaming`; Rust invokes `paimon_cpp_stream_release` +/// when the reader handle is freed, which `delete`s this struct. +/// +/// `pread_mu` is a defensive per-ctx lock: `InputStream::Read(buffer, size, +/// offset)` is declared pread-style (thread-safe, no position mutation), but a +/// few subclasses (notably `JindoInputStream`) have member-variable races. Rust +/// already serializes reads via its own `stream_mutex`; `pread_mu` is a +/// redundant safeguard at the C++ layer. +struct StreamCtx { + std::shared_ptr stream; + std::mutex pread_mu; +}; + +/// `ctx` of `PaimonWriteCallbacks`. Holds a raw (non-owning) pointer to +/// a paimon `OutputStream` plus a sticky error for conveying write failures +/// back to the C++ caller of `TantivyGlobalIndexWriter::Finish`. +struct WriteCtx { + OutputStream* out = nullptr; + Status last_error = Status::OK(); +}; + +/// Rust -> C++ read callback. Reads `len` bytes starting at archive-absolute +/// `offset` into `out_buf`. Returns 0 on success, 1 on IO error. Thread-safe +/// (serialized via `StreamCtx::pread_mu`; Rust also holds its own mutex). +extern "C" int32_t paimon_cpp_stream_read_at(void* ctx_ptr, uint64_t offset, std::size_t len, + uint8_t* out_buf); + +/// Rust -> C++ release callback. Called exactly once when the Rust reader is +/// dropped. Deletes the ctx (which closes the underlying stream via ~shared_ptr). +extern "C" void paimon_cpp_stream_release(void* ctx_ptr); + +/// Rust -> C++ write push callback. Writes `len` bytes from `data` to the +/// underlying OutputStream. Returns 0 on success, 1 on IO error (with the +/// detailed Status stashed in `WriteCtx::last_error` for the caller to pick up). +extern "C" int32_t paimon_cpp_writer_push(void* ctx_ptr, const uint8_t* data, std::size_t len); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp new file mode 100644 index 00000000..36ca7a3a --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Helper: build an archive with `n` documents, return the GlobalIndexIOMeta. +/// Holds the tmp dir alive (via `holder`) so it's cleaned up when the +/// WriteResult goes out of scope. +struct WriteResult { + std::unique_ptr holder; + std::string root_dir; + GlobalIndexIOMeta meta; +}; + +class StreamingTestFixture : public ::testing::Test { + public: + WriteResult BuildArchive(std::size_t n_docs, + const std::string& text_template = "apple banana cherry %zu") { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(root_dir); + std::string root = root_dir->Str(); + + // Build arrow StringArray + arrow::StringBuilder sb; + for (std::size_t i = 0; i < n_docs; ++i) { + char buf[128]; + std::snprintf(buf, sizeof(buf), text_template.c_str(), i); + EXPECT_TRUE(sb.Append(buf).ok()); + } + auto text_array = sb.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({text_array}, {arrow::field("f0", arrow::utf8())}) + .ValueOrDie(); + + std::map options; + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN(auto w, + global_index->CreateWriter("f0", c_schema.get(), file_writer, pool_)); + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*struct_array, &c_array).ok()); + std::vector relative_row_ids(struct_array->length()); + for (int64_t i = 0; i < struct_array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + EXPECT_OK_AND_ASSIGN(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + + // Move root_dir into the result — it stays alive as long as the + // caller holds WriteResult; cleaned up when TEST_F scope exits. + return WriteResult{std::move(root_dir), std::move(root), metas[0]}; + } + + std::shared_ptr OpenReader(const std::string& root, + const GlobalIndexIOMeta& meta) { + std::map options; + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN( + auto reader, global_index->CreateReader(c_schema.get(), file_reader, {meta}, pool_)); + return reader; + } + + std::shared_ptr BuildMatchAll(const std::string& query) { + return std::make_shared( + /*_field_name=*/"f0", + /*_limit=*/std::optional{}, + /*_query=*/query, + /*_search_type=*/FullTextSearch::SearchType::MATCH_ALL, + /*_pre_filter=*/std::optional{}); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +// ========================================================================= +// 1. ParseArchiveHeader fuzz +// ========================================================================= + +TEST(ParseArchiveHeaderFuzz, TruncatedHeader) { + // Fewer than 4 bytes → DataInputStream::ReadValue fails + std::string bytes = "\x00\x00"; + ByteArrayInputStream in(bytes.data(), bytes.size()); + ASSERT_NOK(ArchiveLayout::Parse(&in)) << "expected failure on truncated header"; +} + +TEST(ParseArchiveHeaderFuzz, NegativeFileCount) { + // BE int32 -1 = 0xFFFFFFFF + char bytes[4] = {static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0xFF)}; + ByteArrayInputStream in(bytes, 4); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad file_count"); +} + +TEST(ParseArchiveHeaderFuzz, NameLenOutOfRange) { + // file_count=1, name_len=2GB (BE int32 0x7FFFFFFF) + char bytes[8] = {0, + 0, + 0, + 1, + static_cast(0x7F), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF)}; + ByteArrayInputStream in(bytes, 8); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad name_len"); +} + +TEST(ParseArchiveHeaderFuzz, ZeroFileCountSucceeds) { + // file_count=0 is structurally valid; caller will fail later when + // tantivy::Index::open finds no meta.json, but parse itself OK. + char bytes[4] = {0, 0, 0, 0}; + ByteArrayInputStream in(bytes, 4); + ASSERT_OK_AND_ASSIGN(auto r, ArchiveLayout::Parse(&in)); + ASSERT_EQ(r.count, 0u); +} + +TEST(ParseArchiveHeaderFuzz, PayloadLenNegative) { + // file_count=1, name_len=1, name="a", data_len=-1 (BE int64 0xFFFFFFFFFFFFFFFF) + char bytes[4 + 4 + 1 + 8] = { + // file_count=1 + 0, + 0, + 0, + 1, + // name_len=1 + 0, + 0, + 0, + 1, + // name='a' + 'a', + // data_len = -1 (BE int64 0xFFFFFFFFFFFFFFFF) + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + }; + ByteArrayInputStream in(bytes, sizeof(bytes)); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad data_len"); +} + +// ========================================================================= +// 2. Concurrent query on same reader +// ========================================================================= + +TEST_F(StreamingTestFixture, ConcurrentQueryOnSameReader) { + // 50 docs containing "apple" in every one (all should match) + auto wr = BuildArchive(50, "apple banana %zu"); + auto reader = OpenReader(wr.root_dir, wr.meta); + + auto fts = BuildMatchAll("apple"); + + // 4 threads × 20 queries each, all must return 50 rowIds + constexpr int32_t kThreads = 4; + constexpr int32_t kIters = 20; + std::vector threads; + std::atomic failures{0}; + for (int32_t t = 0; t < kThreads; ++t) { + threads.emplace_back([&] { + for (int32_t i = 0; i < kIters; ++i) { + auto result = reader->VisitFullTextSearch(fts); + if (!result.ok() || !result.value()) { + failures++; + continue; + } + std::shared_ptr r = result.value(); + auto plain = std::dynamic_pointer_cast(r); + if (!plain) { + failures++; + continue; + } + auto bres = plain->GetBitmap(); + if (!bres.ok() || bres.value() == nullptr || bres.value()->Cardinality() != 50) { + failures++; + } + } + }); + } + for (auto& th : threads) { + th.join(); + } + ASSERT_EQ(failures.load(), 0) << "concurrent queries produced inconsistent results"; +} + +// ========================================================================= +// 3. Concurrent reader open + close +// ========================================================================= + +TEST_F(StreamingTestFixture, ConcurrentCreateAndDropReaders) { + // One archive, many readers opening/closing it concurrently. + // Validates exactly-once release (no UAF under ASAN) and open/close race safety. + auto wr = BuildArchive(20); + + constexpr int32_t kThreads = 10; + std::vector threads; + std::atomic failures{0}; + for (int32_t t = 0; t < kThreads; ++t) { + threads.emplace_back([&, t] { + for (int32_t i = 0; i < 5; ++i) { + auto reader = OpenReader(wr.root_dir, wr.meta); + if (!reader) { + failures++; + continue; + } + auto fts = BuildMatchAll("apple"); + auto r = reader->VisitFullTextSearch(fts); + if (!r.ok()) { + failures++; + } + // reader drops here → Rust Arc::drop → paimon_cpp_stream_release + } + (void)t; + }); + } + for (auto& th : threads) { + th.join(); + } + ASSERT_EQ(failures.load(), 0); +} + +} // namespace +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp b/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp new file mode 100644 index 00000000..d82be514 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time for this test" +#endif + +namespace paimon::tantivy::test { +namespace { + +/// Parse the FFI `tokenize` output (tab-separated: from\tto\tpos\ttext\n) and +/// return only the token text sequence. +std::vector ExtractTokenTexts(const PaimonTantivyBuffer& buf) { + std::vector out; + if (buf.len == 0) { + return out; + } + std::string s(reinterpret_cast(buf.data), buf.len); + std::istringstream in(s); + std::string row; + while (std::getline(in, row)) { + // extract text field = after 3rd '\t' + size_t p1 = row.find('\t'); + if (p1 == std::string::npos) { + continue; + } + size_t p2 = row.find('\t', p1 + 1); + if (p2 == std::string::npos) { + continue; + } + size_t p3 = row.find('\t', p2 + 1); + if (p3 == std::string::npos) { + continue; + } + out.emplace_back(row.substr(p3 + 1)); + } + return out; +} + +std::vector TokenizeWithTantivy(PaimonJiebaTokenizer* tok, const std::string& text) { + BufferGuard buf; + PaimonTantivyStatus st = + paimon_tantivy_tokenizer_tokenize(tok, text.data(), text.size(), buf.out()); + EXPECT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK) + << "FFI tokenize failed: " << paimon_tantivy_last_error(); + return ExtractTokenTexts(*buf.out()); +} + +} // namespace + +TEST(TantivyTokenizer, HmmModeReturnsUnsupported) { + std::string dict_dir = JIEBA_TEST_DICT_DIR; + PaimonJiebaTokenizer* handle = nullptr; + PaimonTantivyStatus st = + paimon_tantivy_tokenizer_new("hmm", /*with_position=*/true, dict_dir.c_str(), &handle); + ASSERT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_UNSUPPORTED); + ASSERT_EQ(handle, nullptr); + std::string err = paimon_tantivy_last_error(); + ASSERT_NE(err.find("hmm"), std::string::npos); +} + +// ---------------- positive jieba-rs behavior assertions ---------------- +// +// We do NOT require byte-level parity with cppjieba (the two backends coexist +// and each reads only its own index). Instead assert jieba-rs produces the +// expected token sequence for a curated set of inputs. + +struct JiebaRsCase { + std::string mode; + std::string input; + std::vector expected; +}; + +class JiebaRsBehavior : public ::testing::TestWithParam {}; + +TEST_P(JiebaRsBehavior, ProducesExpectedTokens) { + const auto& c = GetParam(); + std::string dict_dir = JIEBA_TEST_DICT_DIR; + PaimonJiebaTokenizer* handle = nullptr; + PaimonTantivyStatus st = paimon_tantivy_tokenizer_new(c.mode.c_str(), /*with_position=*/true, + dict_dir.c_str(), &handle); + ASSERT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK) << paimon_tantivy_last_error(); + auto got = TokenizeWithTantivy(handle, c.input); + ASSERT_EQ(got, c.expected) << "mode=" << c.mode << " input=" << c.input; + paimon_tantivy_tokenizer_free(handle); +} + +INSTANTIATE_TEST_SUITE_P( + BasicCases, JiebaRsBehavior, + ::testing::Values(JiebaRsCase{"mix", "Hello World", {"hello", "world"}}, + JiebaRsCase{"mix", "HELLO", {"hello"}}, + JiebaRsCase{"mix", "中国人民", {"中国", "人民"}}, + // the two single-char stop words in the input are in + // stop_words.utf8, so Normalize drops them from the output + JiebaRsCase{"mix", "他来到了网易杭研大厦", {"来到", "网易", "杭研", "大厦"}}, + JiebaRsCase{"full", "中国", {"中", "中国", "国"}}, + JiebaRsCase{"query", "中国人民", {"中国", "人民"}})); + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_writer_test.cpp b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp new file mode 100644 index 00000000..0d623b88 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Read the entire file at `path` into a byte buffer. +std::vector ReadFile(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << "open " << path; + in.seekg(0, std::ios::end); + auto sz = static_cast(in.tellg()); + in.seekg(0, std::ios::beg); + std::vector buf(sz); + in.read(reinterpret_cast(buf.data()), sz); + return buf; +} + +/// Read a big-endian integer from a raw pointer. +template +T ReadBE(const uint8_t* p) { + T v = 0; + for (std::size_t i = 0; i < sizeof(T); ++i) { + v = static_cast((v << 8) | static_cast(p[i])); + } + return v; +} + +struct PackedEntry { + std::string name; + int64_t length = 0; + std::size_t offset = 0; // offset in the buffer where bytes start +}; + +/// Parse the packing header into a list of entries; verifies that the offsets +/// and lengths cover the full buffer with no leftover bytes. +/// Format (Java-compatible, big-endian, no version header): +/// [i32 BE file_count | (i32 BE name_len | name | i64 BE file_len | bytes)*] +std::vector ParsePacked(const std::vector& bytes) { + std::vector entries; + EXPECT_GE(bytes.size(), 4u); + auto file_count = ReadBE(bytes.data()); + EXPECT_GT(file_count, 0); + std::size_t off = 4; + for (int32_t i = 0; i < file_count; ++i) { + EXPECT_LE(off + 4, bytes.size()); + auto nlen = ReadBE(bytes.data() + off); + off += 4; + EXPECT_GT(nlen, 0); + EXPECT_LE(off + static_cast(nlen), bytes.size()); + std::string name(reinterpret_cast(bytes.data() + off), + static_cast(nlen)); + off += nlen; + EXPECT_LE(off + 8, bytes.size()); + auto flen = ReadBE(bytes.data() + off); + off += 8; + EXPECT_GE(flen, 0); + EXPECT_LE(off + static_cast(flen), bytes.size()); + entries.push_back({name, flen, off}); + off += static_cast(flen); + } + EXPECT_EQ(off, bytes.size()) << "trailing bytes after pack"; + return entries; +} + +class TantivyGlobalIndexWriterTest : public ::testing::Test { + public: + Result> WriteIndex( + const std::string& root, const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array) { + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE(auto writer, TantivyGlobalIndexWriter::Create( + "f0", data_type, file_writer, options, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(relative_row_ids))); + return writer->Finish(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + std::shared_ptr data_type_ = + arrow::struct_({arrow::field("f0", arrow::utf8())}); +}; + +} // namespace + +TEST_F(TantivyGlobalIndexWriterTest, EnglishCorpusProducesValidPackedIndex) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {kTantivyWriteOmitTermFreqAndPositions, "false"}, + }; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["This is an test document."], + ["This is an new document document document."], + ["Document document document document test."], + ["unordered user-defined doc id"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); + const auto& meta = metas[0]; + + auto file_name = PathUtil::GetName(meta.file_path); + ASSERT_TRUE(StringUtils::StartsWith(file_name, "tantivy-fulltext-global-index-")) + << "file_name=" << file_name; + ASSERT_TRUE(StringUtils::EndsWith(file_name, ".index")); + ASSERT_TRUE(meta.metadata); + ASSERT_EQ(std::string(meta.metadata->data(), meta.metadata->size()), + R"({"write.omit-term-freq-and-position":"false"})"); + ASSERT_GT(meta.file_size, 8); + + auto bytes = ReadFile(meta.file_path); + ASSERT_EQ(static_cast(bytes.size()), meta.file_size); + auto entries = ParsePacked(bytes); + ASSERT_FALSE(entries.empty()); + bool has_meta_json = false; + for (const auto& e : entries) { + if (e.name == "meta.json") { + has_meta_json = true; + } + } + ASSERT_TRUE(has_meta_json) << "expected meta.json in packed entries"; +} + +TEST_F(TantivyGlobalIndexWriterTest, ChineseCorpusProducesValidPackedIndex) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {kTantivyWriteOmitTermFreqAndPositions, "false"}, + {kTantivyWriteTokenizer, "paimon_jieba"}, + {kJiebaTokenizeMode, "query"}, + }; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["千问是一个智能助手"], + ["新一代AI助手发布"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); + const auto& meta = metas[0]; + auto bytes = ReadFile(meta.file_path); + ASSERT_EQ(static_cast(bytes.size()), meta.file_size); + auto entries = ParsePacked(bytes); + ASSERT_FALSE(entries.empty()); +} + +TEST_F(TantivyGlobalIndexWriterTest, NullStringRowsBecomeEmptyDocuments) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["nonempty"], + [null], + ["another"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); +} + +TEST_F(TantivyGlobalIndexWriterTest, RejectsHmmTokenizeMode) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + auto path_factory = std::make_shared(root_dir->Str()); + auto file_writer = std::make_shared(fs_, path_factory); + // hmm rejection only fires when the jieba tokenizer is actually constructed, + // so this test must explicitly opt into jieba (default tokenizer skips + // jieba construction entirely). + std::map options = { + {kTantivyWriteTokenizer, "paimon_jieba"}, + {kJiebaTokenizeMode, "hmm"}, + }; + auto res = TantivyGlobalIndexWriter::Create("f0", data_type_, file_writer, options, pool_); + ASSERT_FALSE(res.ok()); + ASSERT_TRUE(res.status().IsNotImplemented()) << res.status().ToString(); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/testing/mock/mock_file_batch_reader.h b/src/paimon/testing/mock/mock_file_batch_reader.h index 40a7bcb9..f05a2347 100644 --- a/src/paimon/testing/mock/mock_file_batch_reader.h +++ b/src/paimon/testing/mock/mock_file_batch_reader.h @@ -19,7 +19,11 @@ #pragma once #include +#include +#include +#include #include +#include #include #include @@ -28,8 +32,8 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/date_time_utils.h" #include "paimon/reader/prefetch_file_batch_reader.h" + namespace paimon::test { class MockFileBatchReader : public PrefetchFileBatchReader { @@ -41,13 +45,14 @@ class MockFileBatchReader : public PrefetchFileBatchReader { file_schema_(file_schema), read_schema_(arrow::schema(file_schema->fields())), batch_size_(read_batch_size) { + assert(read_batch_size > 0); // add all valid bitmap - int32_t data_length = data_ ? data_->length() : 0; + int64_t data_length = data_ ? data_->length() : 0; + assert(data_length >= 0); + assert(data_length <= static_cast(std::numeric_limits::max())); bitmap_ = RoaringBitmap32(); - bitmap_.AddRange(0, data_length); - read_end_pos_ = data_length; - int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); - std::srand(seed); + bitmap_.AddRange(0, static_cast(data_length)); + read_end_pos_ = static_cast(data_length); } MockFileBatchReader(const std::shared_ptr& data, @@ -95,7 +100,8 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } Status SeekToRow(uint64_t row_number) override { - current_pos_ = row_number; + assert(row_number <= static_cast(std::numeric_limits::max())); + current_pos_ = static_cast(row_number); return Status::OK(); } @@ -119,7 +125,8 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } int32_t actual_batch_size = batch_size_; if (enable_randomize_batch_size_) { - actual_batch_size = std::rand() % batch_size_ + 1; + std::uniform_int_distribution distribution(1, batch_size_); + actual_batch_size = distribution(random_engine_); } int32_t batch_end_pos = std::min(read_end_pos_, current_pos_ + actual_batch_size); auto slice = data_->Slice(current_pos_, batch_end_pos - current_pos_); @@ -151,15 +158,18 @@ class MockFileBatchReader : public PrefetchFileBatchReader { return metrics; } - Result GetPreviousBatchFirstRowNumber() const override { - return previous_batch_first_row_num_; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (previous_batch_first_row_num_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet"); + } + return previous_batch_first_row_num_ + batch_row_id; } Result GetNumberOfRows() const override { - return data_ ? data_->length() : 0; + return ToReaderRowNumber(read_end_pos_); } uint64_t GetNextRowToRead() const override { - return current_pos_; + return ToReaderRowNumber(current_pos_); } void Close() override {} @@ -172,6 +182,13 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } private: + static uint64_t ToReaderRowNumber(int32_t row_number) { + if (row_number < 0) { + return std::numeric_limits::max(); + } + return static_cast(row_number); + } + std::shared_ptr data_; std::shared_ptr file_schema_; std::shared_ptr read_schema_; @@ -179,10 +196,11 @@ class MockFileBatchReader : public PrefetchFileBatchReader { int32_t batch_size_ = 0; int32_t current_pos_ = 0; int32_t read_end_pos_ = 0; - int32_t previous_batch_first_row_num_ = -1; + uint64_t previous_batch_first_row_num_ = std::numeric_limits::max(); Status next_batch_status_; bool enable_randomize_batch_size_ = true; std::vector> read_ranges_; + std::mt19937 random_engine_{std::random_device{}()}; // NOLINT(whitespace/braces) }; } // namespace paimon::test diff --git a/src/paimon/testing/mock/mock_file_system.h b/src/paimon/testing/mock/mock_file_system.h index e0164f1b..484ed1b3 100644 --- a/src/paimon/testing/mock/mock_file_system.h +++ b/src/paimon/testing/mock/mock_file_system.h @@ -37,13 +37,13 @@ class MockInputStream : public InputStream { Result GetPos() const override { return 0; } - Result Read(char* buffer, uint32_t size) override { + Result Read(char* buffer, int64_t size) override { return 0; } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { + Result Read(char* buffer, int64_t size, int64_t offset) override { return 0; } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override {} Status Close() override { @@ -52,7 +52,7 @@ class MockInputStream : public InputStream { Result GetUri() const override { return std::string(); } - Result Length() const override { + Result Length() const override { return 0; } }; @@ -65,7 +65,7 @@ class MockOutputStream : public OutputStream { Result GetPos() const override { return 0; } - Result Write(const char* buffer, uint32_t size) override { + Result Write(const char* buffer, int64_t size) override { return 0; } Status Flush() override { @@ -87,7 +87,7 @@ class MockFileStatus : public FileStatus { std::string GetPath() const override { return ""; } - uint64_t GetLen() const override { + int64_t GetLen() const override { return 0; } int64_t GetModificationTime() const override { diff --git a/src/paimon/testing/mock/mock_format_reader_builder.h b/src/paimon/testing/mock/mock_format_reader_builder.h index 7f8906cd..16819611 100644 --- a/src/paimon/testing/mock/mock_format_reader_builder.h +++ b/src/paimon/testing/mock/mock_format_reader_builder.h @@ -19,7 +19,6 @@ #pragma once #include -#include #include "arrow/type.h" #include "paimon/format/reader_builder.h" diff --git a/src/paimon/testing/mock/mock_format_writer.cpp b/src/paimon/testing/mock/mock_format_writer.cpp index 07feccdd..c7829f46 100644 --- a/src/paimon/testing/mock/mock_format_writer.cpp +++ b/src/paimon/testing/mock/mock_format_writer.cpp @@ -18,8 +18,8 @@ #include "paimon/testing/mock/mock_format_writer.h" +#include #include -#include #include "arrow/c/helpers.h" #include "paimon/common/utils/date_time_utils.h" @@ -35,13 +35,13 @@ class MemoryPool; namespace paimon::test { MockFormatWriter::MockFormatWriter(const std::shared_ptr& out, const std::shared_ptr& pool) - : FormatWriter(), out_(std::move(out)), pool_(pool) {} + : FormatWriter(), out_(out), pool_(pool) {} Status MockFormatWriter::AddBatch(ArrowArray* batch) { ArrowArrayRelease(batch); std::string str = std::to_string(DateTimeUtils::GetCurrentUTCTimeUs()) + "\n"; - PAIMON_ASSIGN_OR_RAISE(int32_t res, out_->Write(str.data(), str.size())); - if (res != static_cast(str.size())) { + PAIMON_ASSIGN_OR_RAISE(int64_t res, out_->Write(str.data(), str.size())); + if (res != static_cast(str.size())) { return Status::IOError("write size does not match"); } counter_++; @@ -60,4 +60,8 @@ Result MockFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return false; } +Status MockFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by mock format writer."); +} + } // namespace paimon::test diff --git a/src/paimon/testing/mock/mock_format_writer.h b/src/paimon/testing/mock/mock_format_writer.h index e395e142..a97a7593 100644 --- a/src/paimon/testing/mock/mock_format_writer.h +++ b/src/paimon/testing/mock/mock_format_writer.h @@ -19,7 +19,9 @@ #pragma once #include +#include #include +#include #include "paimon/format/format_writer.h" #include "paimon/result.h" @@ -46,6 +48,7 @@ class MockFormatWriter : public FormatWriter { std::shared_ptr GetWriterMetrics() const override { return nullptr; } + Status AddMetadata(const std::map& metadata) override; private: int64_t counter_ = 0; diff --git a/src/paimon/testing/mock/mock_format_writer_builder.h b/src/paimon/testing/mock/mock_format_writer_builder.h index d61867f4..a61567d6 100644 --- a/src/paimon/testing/mock/mock_format_writer_builder.h +++ b/src/paimon/testing/mock/mock_format_writer_builder.h @@ -44,8 +44,6 @@ class MockFormatWriterBuilder : public WriterBuilder { const std::string& compression) override; private: - Status Prepare(); - std::shared_ptr memory_pool_; }; diff --git a/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h b/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h index ec8d615c..a2cbfda4 100644 --- a/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h +++ b/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h @@ -23,6 +23,7 @@ #include "paimon/common/data/columnar/columnar_batch_context.h" #include "paimon/core/io/key_value_data_file_record_reader.h" + namespace paimon::test { // mock reader hold data array class MockKeyValueDataFileRecordReader : public KeyValueDataFileRecordReader { diff --git a/src/paimon/testing/utils/CMakeLists.txt b/src/paimon/testing/utils/CMakeLists.txt index 453f963f..25f99d3f 100644 --- a/src/paimon/testing/utils/CMakeLists.txt +++ b/src/paimon/testing/utils/CMakeLists.txt @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -if(PAIMON_BUILD_TESTS) +if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) set(PAIMON_TEST_UTILS testharness.cpp data_generator.cpp) @@ -29,14 +29,25 @@ if(PAIMON_BUILD_TESTS) paimon_static ${GTEST_LINK_TOOLCHAIN}) + # testharness.cpp includes , but the objlib compile step does + # not link the gtest targets, so it has no ordering dependency on the + # googletest ExternalProject. Without this it can race ahead and compile + # before googletest_ep extracts its headers (observed as a flaky + # "gtest/gtest.h: No such file" in Release builds). Force the headers to + # exist before compiling test_utils. + if(TARGET googletest_ep) + add_dependencies(test_utils_objlib googletest_ep) + endif() +endif() + +if(PAIMON_BUILD_TESTS) + add_paimon_test(test_utils_test SOURCES data_generator_test.cpp STATIC_LINK_LIBS paimon_shared - "-Wl,--whole-archive" - paimon_local_file_system_shared - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN}) diff --git a/src/paimon/testing/utils/counting_cache_test_utils.h b/src/paimon/testing/utils/counting_cache_test_utils.h new file mode 100644 index 00000000..bc22fee3 --- /dev/null +++ b/src/paimon/testing/utils/counting_cache_test_utils.h @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/cache/cache.h" +#include "paimon/common/io/cache/lru_cache.h" +#include "paimon/result.h" + +namespace paimon::test { + +class CountingRoutingCache : public Cache { + public: + CountingRoutingCache(CacheKind kind, int64_t max_weight) { + caches_[kind] = std::make_shared(max_weight); + } + + explicit CountingRoutingCache(const std::map& max_weights) { + for (const auto& [kind, max_weight] : max_weights) { + caches_[kind] = std::make_shared(max_weight); + } + } + + Result> Get( + const std::shared_ptr& key, + std::function>(const std::shared_ptr&)> + supplier) override { + CacheKind kind = key->GetKind(); + { + std::lock_guard lock(count_mutex_); + ++get_count_; + last_kind_ = kind; + ++get_count_by_kind_[kind]; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); + return cache->Get( + key, + [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) + -> Result> { + { + std::lock_guard lock(count_mutex_); + ++supplier_call_count_; + ++supplier_call_count_by_kind_[supplier_key->GetKind()]; + } + return supplier(supplier_key); + }); + } + + Status Put(const std::shared_ptr& key, + const std::shared_ptr& value) override { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); + return cache->Put(key, value); + } + + void Invalidate(const std::shared_ptr& key) override { + Result> cache = GetCache(key); + if (cache.ok()) { + cache.value()->Invalidate(key); + } + } + + void InvalidateAll() override { + for (const auto& [kind, cache] : caches_) { + cache->InvalidateAll(); + } + } + + size_t Size() const override { + size_t size = 0; + for (const auto& [kind, cache] : caches_) { + size += cache->Size(); + } + return size; + } + + int64_t GetCount() const { + std::lock_guard lock(count_mutex_); + return get_count_; + } + + int64_t GetCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); + return GetCount(get_count_by_kind_, kind); + } + + int64_t SupplierCallCount() const { + std::lock_guard lock(count_mutex_); + return supplier_call_count_; + } + + int64_t SupplierCallCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); + return GetCount(supplier_call_count_by_kind_, kind); + } + + CacheKind LastKind() const { + std::lock_guard lock(count_mutex_); + return last_kind_; + } + + private: + static int64_t GetCount(const std::map& counts, CacheKind kind) { + auto iter = counts.find(kind); + if (iter == counts.end()) { + return 0; + } + return iter->second; + } + + Result> GetCache(const std::shared_ptr& key) const { + auto iter = caches_.find(key->GetKind()); + if (iter == caches_.end()) { + return Status::Invalid("unexpected cache kind"); + } + return iter->second; + } + + std::map> caches_; + std::map get_count_by_kind_; + std::map supplier_call_count_by_kind_; + int64_t get_count_ = 0; + int64_t supplier_call_count_ = 0; + CacheKind last_kind_ = CacheKind::DEFAULT; + mutable std::mutex count_mutex_; +}; + +} // namespace paimon::test diff --git a/src/paimon/testing/utils/dict_array_converter.h b/src/paimon/testing/utils/dict_array_converter.h index e24b70ec..14ee83be 100644 --- a/src/paimon/testing/utils/dict_array_converter.h +++ b/src/paimon/testing/utils/dict_array_converter.h @@ -30,7 +30,8 @@ class DictArrayConverter { DictArrayConverter() = delete; ~DictArrayConverter() = delete; - // deep copy dictionary array to string array/binary array + // Decode dictionary string arrays to plain StringArray so test comparisons are stable across + // Arrow dictionary index types and string/large_string dictionary values. static Result> ConvertDictArray( const std::shared_ptr& array, arrow::MemoryPool* pool) { arrow::Type::type kind = array->type_id(); @@ -93,21 +94,15 @@ class DictArrayConverter { auto dict_type = arrow::internal::checked_pointer_cast( dict_array->type()); auto value_type_id = dict_type->value_type()->id(); - auto index_type_id = dict_type->index_type()->id(); - if (value_type_id == arrow::Type::type::STRING && - index_type_id == arrow::Type::type::INT32) { - return ConvertDictionaryArrayToBinaryArray< - arrow::StringArray, arrow::Int32Array, arrow::StringBuilder>(dict_array, - pool); - } else if (value_type_id == arrow::Type::type::LARGE_STRING && - index_type_id == arrow::Type::type::INT64) { - return ConvertDictionaryArrayToBinaryArray< - arrow::LargeStringArray, arrow::Int64Array, arrow::StringBuilder>( - dict_array, pool); + if (value_type_id == arrow::Type::type::STRING) { + return ConvertDictionaryArrayToStringArray(dict_array, + pool); + } else if (value_type_id == arrow::Type::type::LARGE_STRING) { + return ConvertDictionaryArrayToStringArray(dict_array, + pool); } else { return Status::Invalid( - "only support [STRING, INT32] or [LARGE_STRING, INT64] for " - "DictionaryArray"); + "only support STRING or LARGE_STRING value type for DictionaryArray"); } } default: { @@ -117,23 +112,25 @@ class DictArrayConverter { } private: - template - static Result> ConvertDictionaryArrayToBinaryArray( + template + static Result> ConvertDictionaryArrayToStringArray( const std::shared_ptr& dict_array, arrow::MemoryPool* pool) { auto dictionary = std::dynamic_pointer_cast(dict_array->dictionary()); - auto indices = std::dynamic_pointer_cast(dict_array->indices()); - auto string_builder = std::make_shared(pool); + if (!dictionary) { + return Status::Invalid("dictionary value array type does not match dictionary type"); + } + + arrow::StringBuilder string_builder(pool); for (int64_t i = 0; i < dict_array->length(); ++i) { if (dict_array->IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.AppendNull()); } else { - int64_t dict_index = indices->Value(i); PAIMON_RETURN_NOT_OK_FROM_ARROW( - string_builder->Append(dictionary->GetString(dict_index))); + string_builder.Append(dictionary->GetString(dict_array->GetValueIndex(i)))); } } std::shared_ptr string_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder->Finish(&string_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.Finish(&string_array)); return string_array; } }; diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h index bbfc21ab..8321a838 100644 --- a/src/paimon/testing/utils/read_result_collector.h +++ b/src/paimon/testing/utils/read_result_collector.h @@ -70,10 +70,6 @@ class ReadResultCollector { return results; } - static Result> CollectResult(BatchReader* batch_reader) { - return CollectResult(batch_reader, /*max simulated data processing time*/ 0); - } - // will convert dictionary array to string array for comparing results static Result> CollectResult( BatchReader* batch_reader, int64_t max_data_processing_time_in_us) { @@ -81,35 +77,10 @@ class ReadResultCollector { int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); std::srand(seed); while (true) { - // Prioritize calling NextBatch. If it fails (paimon inner reader e.g., - // PrefetchBatchReader, ApplyBitmapIndexBatchReader...), call NextBatchWithBitmap. - auto batch_result = batch_reader->NextBatch(); - BatchReader::ReadBatch batch; - if (!batch_result.ok()) { - if (batch_result.status().ToString().find("should use NextBatchWithBitmap") != - std::string::npos) { - PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, - batch_reader->NextBatchWithBitmap()); - if (BatchReader::IsEofBatch(batch_with_bitmap)) { - break; - } - assert(!batch_with_bitmap.second.IsEmpty()); - PAIMON_ASSIGN_OR_RAISE( - batch, ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), - arrow::default_memory_pool())); - } else { - return batch_result.status(); - } - } else { - batch = std::move(batch_result).value(); - if (BatchReader::IsEofBatch(batch)) { - break; - } + PAIMON_ASSIGN_OR_RAISE(auto result_array, ReadOneBatch(batch_reader)); + if (result_array == nullptr) { + break; } - auto& [c_array, c_schema] = batch; - assert(c_array->length > 0); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result_array, - arrow::ImportArray(c_array.get(), c_schema.get())); result_array_vector.push_back(result_array); if (max_data_processing_time_in_us > 0) { usleep(std::rand() % max_data_processing_time_in_us); @@ -133,6 +104,35 @@ class ReadResultCollector { return chunk_array; } + static Result> CollectResult(BatchReader* batch_reader) { + return CollectResult(batch_reader, /*max_data_processing_time_in_us=*/0); + } + + static Result> CollectResultOneBatch( + BatchReader* batch_reader) { + return CollectResultOneBatch(batch_reader, /*max_data_processing_time_in_us=*/0); + } + + static Result> CollectResultOneBatch( + BatchReader* batch_reader, int64_t max_data_processing_time_in_us) { + int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); + std::srand(seed); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_array, + ReadOneBatch(batch_reader)); + if (result_array == nullptr) { + return std::shared_ptr(); + } + PAIMON_ASSIGN_OR_RAISE( + auto converted_array, + DictArrayConverter::ConvertDictArray(result_array, arrow::default_memory_pool())); + if (max_data_processing_time_in_us > 0) { + usleep(std::rand() % max_data_processing_time_in_us); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr chunk_array, + arrow::ChunkedArray::Make({converted_array})); + return chunk_array; + } + static Result> GetArray(BatchReader::ReadBatch&& batch) { if (BatchReader::IsEofBatch(batch)) { return std::shared_ptr(); @@ -167,5 +167,58 @@ class ReadResultCollector { arrow::compute::Take(arrow::Datum(array), arrow::Datum(sorted_indices))); return sorted_batch.chunked_array(); } + + private: + static Result> ReadOneBatch(BatchReader* batch_reader) { + // Prioritize calling NextBatch. If it fails (paimon inner reader e.g., + // PrefetchBatchReader, ApplyBitmapIndexBatchReader...), call NextBatchWithBitmap. + auto batch_result = batch_reader->NextBatch(); + BatchReader::ReadBatch batch; + if (!batch_result.ok()) { + if (batch_result.status().ToString().find("should use NextBatchWithBitmap") != + std::string::npos) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + batch_reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return std::shared_ptr(); + } + PAIMON_RETURN_NOT_OK(CheckBatchOffset(batch_with_bitmap.first)); + assert(!batch_with_bitmap.second.IsEmpty()); + PAIMON_ASSIGN_OR_RAISE( + batch, ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), + arrow::default_memory_pool())); + } else { + return batch_result.status(); + } + } else { + batch = std::move(batch_result).value(); + if (BatchReader::IsEofBatch(batch)) { + return std::shared_ptr(); + } + PAIMON_RETURN_NOT_OK(CheckBatchOffset(batch)); + } + auto& [c_array, c_schema] = batch; + assert(c_array->length > 0); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + return result_array; + } + + static Status CheckBatchOffset(const BatchReader::ReadBatch& batch) { + assert(!BatchReader::IsEofBatch(batch)); + return CheckArrayOffset(batch.first.get()); + } + + static Status CheckArrayOffset(const ArrowArray* array) { + assert(array); + if (array->offset != 0) { + return Status::Invalid("BatchReader returned an array with non-zero offset " + + std::to_string(array->offset)); + } + for (int64_t i = 0; i < array->n_children; i++) { + PAIMON_RETURN_NOT_OK(CheckArrayOffset(array->children[i])); + } + return Status::OK(); + } }; } // namespace paimon::test diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index c6a05092..429c3be9 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -24,6 +24,8 @@ #include #include +#include "arrow/api.h" +#include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/ipc/api.h" #include "paimon/api.h" @@ -59,25 +61,22 @@ class TestHelper { const std::vector& partition_keys, const std::vector& primary_keys, const std::map& options, bool is_streaming_mode, - bool ignore_if_exists = false) { - // only for test && only check the key - auto new_options = options; - new_options["enable-object-store-catalog-in-inte-test"] = ""; - PAIMON_ASSIGN_OR_RAISE(auto catalog, Catalog::Create(root_path, new_options)); - PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", new_options, ignore_if_exists)); + bool ignore_if_exists = false, const std::string& temp_directory = "") { + PAIMON_ASSIGN_OR_RAISE(auto catalog, Catalog::Create(root_path, options)); + PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, ignore_if_exists)); ::ArrowSchema c_schema; ScopeGuard guard([schema = &c_schema]() { ArrowSchemaRelease(schema); }); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); PAIMON_RETURN_NOT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, - partition_keys, primary_keys, new_options, + partition_keys, primary_keys, options, ignore_if_exists)); std::string table_path = PathUtil::JoinPath(root_path, "foo.db/bar"); - return Create(table_path, new_options, is_streaming_mode); + return Create(table_path, options, is_streaming_mode, temp_directory); } static Result> Create( const std::string& table_path, const std::map& options, - bool is_streaming_mode) { + bool is_streaming_mode, const std::string& temp_directory = "") { std::string file_system_identifier = "local"; auto fs_iter = options.find(Options::FILE_SYSTEM); if (fs_iter != options.end()) { @@ -87,6 +86,9 @@ class TestHelper { FileSystemFactory::Get(file_system_identifier, table_path, options)); std::string commit_user = "commit_user"; WriteContextBuilder context_builder(table_path, commit_user); + if (!temp_directory.empty()) { + context_builder.WithTempDirectory(temp_directory); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.SetOptions(options) .WithStreamingMode(is_streaming_mode) @@ -94,14 +96,10 @@ class TestHelper { .Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, FileStoreWrite::Create(std::move(write_context))); - std::map new_options = options; - // only for test && only check the key - new_options["enable-pk-commit-in-inte-test"] = ""; - new_options["enable-object-store-commit-in-inte-test"] = ""; CommitContextBuilder commit_context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit_context, - commit_context_builder.SetOptions(new_options).IgnoreEmptyCommit(false).Finish()); + commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, FileStoreCommit::Create(std::move(commit_context))); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), @@ -174,6 +172,27 @@ class TestHelper { return result_plan->Splits(); } + /// Builds a one-row struct array holding the serialized descriptor of the blob. + static Result> MakeBlobDescriptorArray( + const std::shared_ptr& struct_type, const std::shared_ptr& blob, + const std::shared_ptr& pool) { + if (struct_type->num_fields() != 1 || + struct_type->field(0)->type()->id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid("struct_type must have a single large binary field"); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(descriptor->data(), descriptor->size())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); + return array; + } + static Result CheckBlobsEqual(const std::vector>& result_blobs, const std::vector>& expected_blobs, const std::shared_ptr& fs) { @@ -186,8 +205,8 @@ class TestHelper { PAIMON_ASSIGN_OR_RAISE(auto result_stream, result_blobs[i]->NewInputStream(fs)); PAIMON_ASSIGN_OR_RAISE(auto expected_stream, expected_blobs[i]->NewInputStream(fs)); - PAIMON_ASSIGN_OR_RAISE(uint64_t result_length, result_stream->Length()); - PAIMON_ASSIGN_OR_RAISE(uint64_t expected_length, expected_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t result_length, result_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t expected_length, expected_stream->Length()); if (result_length != expected_length) { auto result_descriptor_bytes = result_blobs[i]->ToDescriptor(GetDefaultPool()); auto expected_descriptor_bytes = expected_blobs[i]->ToDescriptor(GetDefaultPool()); @@ -241,72 +260,38 @@ class TestHelper { return result_blobs; } - // need to reconstruct the blob array, because the array in read result do not have blob meta - Result> ReconstructBlobArray( - const std::shared_ptr& array, const std::shared_ptr& schema) { - ::ArrowArray c_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); - ::ArrowSchema new_c_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &new_c_schema)); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto new_array, - arrow::ImportArray(&c_array, &new_c_schema)); - return new_array; + /// Reads all rows of the given splits and returns the raw result (including the leading + /// `_VALUE_KIND` column). Useful when the expected data cannot be expressed as JSON, e.g. + /// binary-encoded VARIANT columns. + Result> ReadResult( + const std::vector>& splits) { + return ReadResult(splits, /*read_schema=*/nullptr); } - Result ReadAndCheckResultForBlobTable( - const std::shared_ptr& all_columns_schema, - const std::vector>& splits, const std::string& main_expected_json, - const std::vector>& expected_blob_descriptors) { + /// Reads all rows of the given splits with an optional projected read schema. + Result> ReadResult( + const std::vector>& splits, + std::unique_ptr<::ArrowSchema> read_schema) { ReadContextBuilder read_context_builder(table_path_); read_context_builder.SetOptions(options_); + if (read_schema != nullptr) { + read_context_builder.SetReadSchema(std::move(read_schema)); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); - PAIMON_ASSIGN_OR_RAISE(auto read_result, + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, ReadResultCollector::CollectResult(batch_reader.get())); - - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto concat_array, - arrow::Concatenate(read_result->chunks())); - PAIMON_ASSIGN_OR_RAISE(auto reconstruct_array, - ReconstructBlobArray(concat_array, all_columns_schema)); - PAIMON_ASSIGN_OR_RAISE( - auto separated_array, - BlobUtils::SeparateBlobArray( - std::dynamic_pointer_cast(reconstruct_array))); - - arrow::EqualOptions equal_options = arrow::EqualOptions::Defaults(); - - // check main columns - auto separated_schema = BlobUtils::SeparateBlobSchema(all_columns_schema); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto main_expected_array, - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_(separated_schema.main_schema->fields()), main_expected_json)); - auto main_expected_chunk_array = std::make_shared(main_expected_array); - bool main_equal = main_expected_chunk_array->Equals( - arrow::ChunkedArray(separated_array.main_array), equal_options.diff_sink(&std::cout)); - if (!main_equal) { - std::cout << "[expected_data_type]" << main_expected_chunk_array->type()->ToString() - << std::endl; - std::cout << "[actual_data_type]" << separated_array.main_array->type()->ToString() - << std::endl; - std::cout << "[expected]:" << main_expected_chunk_array->ToString() << std::endl; - std::cout << "[actual]: " << separated_array.main_array->ToString() << std::endl; - } - - // check blob column - std::vector> expected_blobs; - for (const auto& descriptor : expected_blob_descriptors) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob, - Blob::FromDescriptor(descriptor->data(), descriptor->size())); - expected_blobs.emplace_back(blob); + if (collected->num_chunks() == 0) { + return collected; } - PAIMON_ASSIGN_OR_RAISE(auto result_blobs, ToBlobs(separated_array.blob_array)); - PAIMON_ASSIGN_OR_RAISE(bool blob_equal, CheckBlobsEqual(result_blobs, expected_blobs, fs_)); - - table_read.reset(); - return main_equal && blob_equal; + // The collected batches borrow reader-owned buffers; copy them into the process pool + // while the reader is still alive so the returned result may outlive it. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr copied, + arrow::Concatenate(collected->chunks(), arrow::default_memory_pool())); + return std::make_shared(copied); } Result ReadAndCheckResult(const std::shared_ptr& data_type, diff --git a/src/paimon/testing/utils/testharness.cpp b/src/paimon/testing/utils/testharness.cpp index 9c61ac05..d018ab08 100644 --- a/src/paimon/testing/utils/testharness.cpp +++ b/src/paimon/testing/utils/testharness.cpp @@ -107,6 +107,8 @@ std::map GetJindoTestOptions() { {"fs.oss.bucket.paimon-unittest.accessKeyId", access_key_id}, {"fs.oss.bucket.paimon-unittest.accessKeySecret", access_key_secret}, {"fs.oss.user", "paimon"}, + {"enable-object-store-catalog-in-inte-test", ""}, + {"enable-object-store-commit-in-inte-test", ""}, }; return options; } diff --git a/src/paimon/testing/utils/testharness.h b/src/paimon/testing/utils/testharness.h index 5c92fa14..0886b517 100644 --- a/src/paimon/testing/utils/testharness.h +++ b/src/paimon/testing/utils/testharness.h @@ -93,15 +93,15 @@ ::testing::AssertionResult AssertStatus(const char* s_expr, const Status& s); #define ASSIGN_OR_HANDLE_ERROR_IMPL(handle_error, status_name, lhs, rexpr) \ auto&& status_name = (rexpr); \ handle_error(status_name.status()); \ - lhs = std::move(status_name).value(); + lhs = std::move(status_name).value() #define ASSERT_OK_AND_ASSIGN(lhs, rexpr) \ ASSIGN_OR_HANDLE_ERROR_IMPL( \ - ASSERT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); + ASSERT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr) #define EXPECT_OK_AND_ASSIGN(lhs, rexpr) \ ASSIGN_OR_HANDLE_ERROR_IMPL( \ - EXPECT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); + EXPECT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr) #define EXPECT_OK(s) EXPECT_PRED_FORMAT1(paimon::test::AssertStatus, s) #define EXPECT_NOK(s) EXPECT_FALSE((s).ok()) diff --git a/src/paimon/testing/utils/variant_test_data.h b/src/paimon/testing/utils/variant_test_data.h new file mode 100644 index 00000000..e78b0cb9 --- /dev/null +++ b/src/paimon/testing/utils/variant_test_data.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon::test { + +/// Helpers for building variant test data. +class VariantTestData { + public: + VariantTestData() = delete; + ~VariantTestData() = delete; + + /// Builds a `[id, v]` struct array where `v` holds the variant encodings of `jsons` (nullptr + /// means a null variant) and `id` is a running int32 starting at `id_offset`. + /// `variant_field` must be a variant-marked field (see `VariantTypeUtils::ToArrowField`). + static Result> BuildVariantBatch( + const std::shared_ptr& id_field, + const std::shared_ptr& variant_field, const std::vector& jsons, + const std::shared_ptr& pool, int32_t id_offset = 0); +}; + +inline Result> VariantTestData::BuildVariantBatch( + const std::shared_ptr& id_field, + const std::shared_ptr& variant_field, const std::vector& jsons, + const std::shared_ptr& pool, int32_t id_offset) { + arrow::Int32Builder id_builder; + auto value_builder = std::make_shared(); + auto metadata_builder = std::make_shared(); + arrow::StructBuilder variant_builder(variant_field->type(), arrow::default_memory_pool(), + {value_builder, metadata_builder}); + for (size_t i = 0; i < jsons.size(); ++i) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(id_offset + static_cast(i))); + if (jsons[i] == nullptr) { + // StructBuilder::AppendNull appends empty values to the child builders itself. + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.AppendNull()); + } else { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::FromJson(jsons[i], pool)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.Append()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(variant->RawValue())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(variant->Metadata())); + } + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr id_array, id_builder.Finish()); + std::shared_ptr variant_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.Finish(&variant_array)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr result, + arrow::StructArray::Make({id_array, variant_array}, {id_field, variant_field})); + return result; +} + +} // namespace paimon::test diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index e9e48146..85fb8f30 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -22,6 +22,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(variant_table_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(data_evolution_table_test STATIC_LINK_LIBS paimon_shared @@ -99,4 +106,18 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(nested_column_pruning_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + + add_paimon_test(predicate_abi_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + endif() diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 88d81151..b8a2c87a 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -21,11 +21,15 @@ #include #include +#include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/io/data_file_meta.h" @@ -36,10 +40,14 @@ #include "paimon/executor.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/read_context.h" #include "paimon/result.h" +#include "paimon/table/source/table_read.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/data_generator.h" #include "paimon/testing/utils/io_exception_helper.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -75,8 +83,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -95,8 +103,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -109,8 +117,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); // @note: for append-only tables in Spark, native row-level deletes aren't supported during // writing. Instead, deletions are expressed by committing a Deletion Vector (DV) file @@ -119,6 +127,7 @@ class AppendCompactionInteTest : public testing::Test, auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); int32_t bucket = 1; auto abstract_write = dynamic_cast(helper->write_.get()); + ASSERT_NE(abstract_write, nullptr); ASSERT_OK_AND_ASSIGN(auto restore_files, abstract_write->ScanExistingFileMetas(partition, bucket)); ASSERT_OK_AND_ASSIGN( @@ -199,8 +208,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -243,6 +252,245 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { } } +TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + auto schema = arrow::schema(fields); + + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_ROW_NUM, "1"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "64"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 40], ["d", 50]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [5, [["e", 60], ["f", 70], ["g", 80], ["h", 90]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(data_splits.size(), 1); + { + // Compaction ignores target-file-row-num and creates one five-row output file. It also + // creates a fresh shared-shredding writer, so the file starts from K_max. + auto data_split = std::dynamic_pointer_cast(data_splits[0]); + ASSERT_TRUE(data_split); + ASSERT_EQ(data_split->DataFiles().size(), 1); + auto compact_file = data_split->DataFiles()[0]; + std::string compact_file_path = + PathUtil::JoinPath(data_split->BucketPath(), compact_file->file_name); + ASSERT_OK_AND_ASSIGN(auto unique_input_stream, + dir->GetFileSystem()->Open(compact_file_path)); + std::shared_ptr input_stream(std::move(unique_input_stream)); + ASSERT_OK_AND_ASSIGN(auto file_format_obj, FileFormatFactory::Get(file_format, options)); + ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format_obj->CreateReaderBuilder(10)); + ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + auto tags_field = file_schema->GetFieldByName("tags"); + ASSERT_TRUE(tags_field); + ASSERT_TRUE(tags_field->metadata()); + ASSERT_OK_AND_ASSIGN(auto tags_meta, MapSharedShreddingUtils::DeserializeMetadata( + tags_field->metadata()->Copy())); + ASSERT_EQ(64, tags_meta.num_columns); + ASSERT_EQ(4, tags_meta.max_row_width); + } + { + // recall all fields + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", 10], ["b", 20]]], + [0, 2, [["c", 30]]], + [0, 3, [["a", 40], ["d", 50]]], + [0, 4, null], + [0, 5, [["e", 60], ["f", 70], ["g", 80], ["h", 90]]] + ])")); + ASSERT_TRUE(success); + } + { + // recall only "a,f" sub-key in map + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,f"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 1, [["a", 10]]], + [0, 2, []], + [0, 3, [["a", 40]]], + [0, 4, null], + [0, 5, [["f", 70]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(expected_chunked->Equals(actual)) + << "actual=" << actual->ToString() << "\nexpected=" << expected_chunked->ToString(); + } +} + +TEST_P(AppendCompactionInteTest, + TestOrcAppendTableFullCompactionWithMapSharedShreddingStringValue) { + auto file_format = GetParam(); + if (file_format != "orc") { + return; + } + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto map_type = arrow::map(arrow::utf8(), arrow::utf8()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + auto schema = arrow::schema(fields); + + std::map options = { + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", "shared"], ["b", "hot"]]], + [2, [["c", "shared"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", "shared"], ["d", "hot"]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [5, [["e", "shared"], ["f", "hot"], ["g", "shared"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(data_splits.size(), 1); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", "shared"], ["b", "hot"]]], + [0, 2, [["c", "shared"]]], + [0, 3, [["a", "shared"], ["d", "hot"]]], + [0, 4, null], + [0, 5, [["e", "shared"], ["f", "hot"], ["g", "shared"]]] + ])")); + ASSERT_TRUE(success); +} + TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -287,8 +535,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) ASSERT_OK(helper2->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper2->LatestSnapshot()); ASSERT_EQ(6, snapshot5.value().Id()); - ASSERT_EQ(8, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper2->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -369,8 +617,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteBestEffortCompaction) ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -460,8 +708,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteCompactionWithExterna ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -520,7 +768,7 @@ TEST_F(AppendCompactionInteTest, TestAppendTableCompactionWithIOException) { bool compaction_run_complete = false; auto io_hook = IOHook::GetInstance(); - for (size_t i = 0; i < 600; ++i) { + for (size_t i = 0; i < 2000; ++i) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index ca6df4f1..fa72f811 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -16,12 +16,17 @@ * limitations under the License. */ +#include #include #include +#include #include #include #include +#include +#include #include +#include #include #include @@ -39,7 +44,12 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" @@ -72,17 +82,25 @@ #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" + namespace paimon { class DataSplit; class RecordBatch; } // namespace paimon namespace paimon::test { + +struct ReadResult { + std::unique_ptr batch_reader; + std::shared_ptr chunked_array; +}; + class BlobTableInteTest : public testing::Test, public ::testing::WithParamInterface { public: void SetUp() override { pool_ = GetDefaultPool(); dir_ = UniqueTestDirectory::Create("local"); + blob_dir_ = UniqueTestDirectory::Create("local"); } void TearDown() override { @@ -91,7 +109,13 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter void CreateTable(const std::vector& partition_keys, const std::map& options) const { - auto schema = arrow::schema(fields_); + CreateTable(fields_, partition_keys, options); + } + + void CreateTable(const arrow::FieldVector& fields, + const std::vector& partition_keys, + const std::map& options) const { + auto schema = arrow::schema(fields); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -151,6 +175,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } } } + Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit @@ -162,11 +187,29 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return file_store_commit->Commit(commit_msgs); } - Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, - const std::shared_ptr& expected_array, - const std::shared_ptr& predicate = nullptr, - const std::vector& row_ranges = {}) const { - // scan + Status WriteNextSchema(const std::string& table_path, const std::vector& fields, + int32_t highest_field_id, + const std::map& options) const { + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->fields_ = fields; + next_schema->highest_field_id_ = highest_field_id; + next_schema->options_ = options; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return dir_->GetFileSystem()->AtomicStore(schema_path, schema_content); + } + + /// Scan table and return the plan (without reading data). + Result> ScanTable(const std::string& table_path, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { @@ -176,47 +219,72 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); - if (!expected_array) { - EXPECT_TRUE(result_plan->Splits().empty()); - } + return result_plan; + } - // read - auto splits = result_plan->Splits(); + /// Read from table using a pre-scanned plan, returning the ChunkedArray and batch_reader. + /// The batch_reader must outlive the returned ChunkedArray (array memory depends on reader). + Result ReadTable(const std::string& table_path, + const std::vector& read_schema, + const std::shared_ptr& plan, + const std::shared_ptr& predicate = nullptr, + const std::map& options = {}) const { + auto splits = plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); + read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); + if (!options.empty()) { + read_context_builder.SetOptions(options); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + return ReadResult{std::move(batch_reader), std::move(read_result)}; + } - if (!expected_array) { - EXPECT_FALSE(read_result); - return Status::OK(); - } - // add row kind array for expected array + /// Convenience: scan + read in one call. + Result ScanAndReadResult(const std::string& table_path, + const std::vector& read_schema, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { + PAIMON_ASSIGN_OR_RAISE(auto result_plan, ScanTable(table_path, predicate, row_ranges)); + return ReadTable(table_path, read_schema, result_plan, predicate); + } + + /// Prepend a _VALUE_KIND (Insert) column to a StructArray. + static Result> PrependRowKindColumn( + const std::shared_ptr& array) { auto row_kind_scalar = std::make_shared(RowKind::Insert()->ToByteValue()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto row_kind_array, - arrow::MakeArrayFromScalar(*row_kind_scalar, expected_array->length())); - arrow::ArrayVector expected_with_row_kind_fields = expected_array->fields(); - std::vector expected_with_row_kind_field_names = - arrow::schema(expected_array->type()->fields())->field_names(); - expected_with_row_kind_fields.insert(expected_with_row_kind_fields.begin(), row_kind_array); - expected_with_row_kind_field_names.insert(expected_with_row_kind_field_names.begin(), - "_VALUE_KIND"); - - // check read result + auto row_kind_array, arrow::MakeArrayFromScalar(*row_kind_scalar, array->length())); + arrow::ArrayVector fields_with_row_kind = array->fields(); + std::vector names_with_row_kind = + arrow::schema(array->type()->fields())->field_names(); + fields_with_row_kind.insert(fields_with_row_kind.begin(), row_kind_array); + names_with_row_kind.insert(names_with_row_kind.begin(), "_VALUE_KIND"); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto expected_with_row_kind_array, - arrow::StructArray::Make(expected_with_row_kind_fields, - expected_with_row_kind_field_names)); - auto expected_chunk_array = - std::make_shared(expected_with_row_kind_array); - EXPECT_TRUE(expected_chunk_array->Equals(read_result)) - << "result:" << read_result->ToString() << std::endl + auto result, arrow::StructArray::Make(fields_with_row_kind, names_with_row_kind)); + return std::dynamic_pointer_cast(result); + } + + Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, + const std::shared_ptr& expected_array, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { + PAIMON_ASSIGN_OR_RAISE(auto scan_read, + ScanAndReadResult(table_path, read_schema, predicate, row_ranges)); + + if (!expected_array) { + EXPECT_FALSE(scan_read.chunked_array); + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(auto expected_with_row_kind, PrependRowKindColumn(expected_array)); + auto expected_chunk_array = std::make_shared(expected_with_row_kind); + EXPECT_TRUE(expected_chunk_array->Equals(scan_read.chunked_array)) + << "result:" << scan_read.chunked_array->ToString() << std::endl << "expected:" << expected_chunk_array->ToString(); return Status::OK(); } @@ -238,15 +306,222 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter .ValueOrDie()); } + /// Convert a StructArray with raw blob bytes into a StructArray with serialized + /// BlobDescriptor bytes. Each raw blob value is written to a temporary file, and + /// the corresponding cell is replaced with the serialized BlobDescriptor pointing + /// to that file. + /// Common framework for transforming blob fields in a StructArray. + /// Non-blob fields are kept as-is; blob fields are processed row-by-row via `transform_row`. + /// `transform_row` receives (binary_value_view) and returns the transformed bytes via builder. + using BlobRowTransform = + std::function; + + Result> TransformBlobFields( + const std::shared_ptr& input_array, + const std::set& blob_fields, BlobRowTransform transform_row) const { + auto fields = input_array->type()->fields(); + arrow::ArrayVector child_arrays; + + for (const auto& field : fields) { + auto col = input_array->GetFieldByName(field->name()); + if (blob_fields.count(field->name()) == 0) { + child_arrays.push_back(col); + continue; + } + const auto& binary_array = + arrow::internal::checked_cast(*col); + arrow::LargeBinaryBuilder builder; + for (int64_t i = 0; i < binary_array.length(); ++i) { + if (binary_array.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK(transform_row(binary_array.GetView(i), &builder)); + } + std::shared_ptr result_col; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result_col)); + child_arrays.push_back(result_col); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result, + arrow::StructArray::Make(child_arrays, fields)); + return result; + } + + Result> ConvertRawBlobToDescriptor( + const std::shared_ptr& raw_array, + const std::set& blob_fields) { + auto fs = std::make_shared(); + return TransformBlobFields( + raw_array, blob_fields, + [&](const std::string_view& raw_value, arrow::LargeBinaryBuilder* builder) -> Status { + std::string file_path = + blob_dir_->Str() + "/blob_" + std::to_string(blob_file_counter_++) + ".bin"; + auto raw_size = static_cast(raw_value.size()); + PAIMON_ASSIGN_OR_RAISE(auto out, fs->Create(file_path, /*overwrite=*/true)); + PAIMON_ASSIGN_OR_RAISE(auto written, out->Write(raw_value.data(), raw_size)); + PAIMON_RETURN_NOT_OK(out->Flush()); + PAIMON_RETURN_NOT_OK(out->Close()); + if (written != raw_size) { + return Status::Invalid("Short write: expected {}, wrote {}", raw_value.size(), + written); + } + PAIMON_ASSIGN_OR_RAISE(auto blob, Blob::FromPath(file_path)); + auto descriptor = blob->ToDescriptor(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(descriptor->data(), descriptor->size())); + return Status::OK(); + }); + } + + /// Delete the file referenced by the serialized BlobDescriptor at `row` of blob field + /// `field_name`, so that a subsequent table write observes a missing file. + Status DeleteDescriptorTarget(const std::shared_ptr& desc_array, + const std::string& field_name, int64_t row) const { + const auto& blob_col = arrow::internal::checked_cast( + *desc_array->GetFieldByName(field_name)); + std::string_view descriptor_bytes = blob_col.GetView(row); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + LocalFileSystem fs; + return fs.Delete(descriptor->Uri(), /*recursive=*/false); + } + + /// Rewrite the serialized BlobDescriptor at `row` of blob field `field_name` to reference + /// an offset beyond the end of the target file, so that a subsequent table write observes + /// a fetch failure that is not a missing file. `row` counts non-null rows. + Result> CorruptDescriptorOffset( + const std::shared_ptr& desc_array, const std::string& field_name, + int64_t row) const { + int64_t current_row = 0; + return TransformBlobFields( + desc_array, {field_name}, + [&](const std::string_view& descriptor_bytes, + arrow::LargeBinaryBuilder* builder) -> Status { + if (current_row++ != row) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(descriptor_bytes.data(), descriptor_bytes.size())); + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr corrupted, + BlobDescriptor::Create(descriptor->Version(), descriptor->Uri(), + /*offset=*/1 << 20, descriptor->Length())); + auto corrupted_bytes = corrupted->Serialize(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(corrupted_bytes->data(), corrupted_bytes->size())); + return Status::OK(); + }); + } + + struct BlobDescriptorPathRewrite { + std::string table_path; + std::vector table_relative_blob_dirs; + }; + + static std::optional TryRewriteDescriptorUri( + const std::string& descriptor_uri, const BlobDescriptorPathRewrite& rewrite, + const std::shared_ptr& fs) { + if (rewrite.table_path.empty()) { + return std::nullopt; + } + + for (const auto& blob_dir : rewrite.table_relative_blob_dirs) { + const std::string marker = "/" + blob_dir + "/"; + auto marker_pos = descriptor_uri.find(marker); + if (marker_pos != std::string::npos) { + std::string relative_blob_path = descriptor_uri.substr(marker_pos + 1); + return PathUtil::JoinPath(rewrite.table_path, relative_blob_path); + } + } + return std::nullopt; + } + + /// Convert a StructArray with serialized BlobDescriptor bytes back to a StructArray + /// with raw blob bytes. Only blob fields are resolved; other columns (including + /// _VALUE_KIND) are kept as-is. + Result> ConvertDescriptorToRawBlob( + const std::shared_ptr& desc_array, + const std::set& blob_fields, + const BlobDescriptorPathRewrite& rewrite = {}) const { + auto fs = std::make_shared(); + return TransformBlobFields( + desc_array, blob_fields, + [&](const std::string_view& descriptor_bytes, + arrow::LargeBinaryBuilder* builder) -> Status { + PAIMON_ASSIGN_OR_RAISE( + auto descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + std::string descriptor_uri = descriptor->Uri(); + auto rewritten_uri = TryRewriteDescriptorUri(descriptor_uri, rewrite, fs); + if (rewritten_uri.has_value()) { + descriptor_uri = rewritten_uri.value(); + } + + PAIMON_ASSIGN_OR_RAISE( + auto rewritten_descriptor, + BlobDescriptor::Create(descriptor->Version(), descriptor_uri, + descriptor->Offset(), descriptor->Length())); + auto rewritten_descriptor_bytes = rewritten_descriptor->Serialize(pool_); + PAIMON_ASSIGN_OR_RAISE(auto blob, + Blob::FromDescriptor(rewritten_descriptor_bytes->data(), + rewritten_descriptor_bytes->size())); + PAIMON_ASSIGN_OR_RAISE(auto data, blob->ToData(fs, pool_)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(data->data(), data->size())); + return Status::OK(); + }); + } + + /// Verify DataFileMeta properties from a scan plan. + /// Each vector element corresponds to one expected DataFileMeta (ordered by file index). + static void VerifyDataFileMetas( + const std::shared_ptr& plan, size_t expected_file_count, + const std::vector& expected_row_counts, + const std::vector& expected_min_seqs, + const std::vector& expected_max_seqs, + const std::vector& expected_first_row_ids, + const std::vector>>& expected_write_cols) { + std::vector> all_files; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + all_files.push_back(file); + } + } + ASSERT_EQ(all_files.size(), expected_file_count); + ASSERT_EQ(expected_row_counts.size(), expected_file_count); + ASSERT_EQ(expected_min_seqs.size(), expected_file_count); + ASSERT_EQ(expected_max_seqs.size(), expected_file_count); + ASSERT_EQ(expected_first_row_ids.size(), expected_file_count); + ASSERT_EQ(expected_write_cols.size(), expected_file_count); + for (size_t i = 0; i < all_files.size(); ++i) { + const auto& file = all_files[i]; + EXPECT_EQ(file->row_count, expected_row_counts[i]); + EXPECT_EQ(file->min_sequence_number, expected_min_seqs[i]); + EXPECT_EQ(file->max_sequence_number, expected_max_seqs[i]); + ASSERT_TRUE(file->first_row_id.has_value()); + EXPECT_EQ(file->first_row_id.value(), expected_first_row_ids[i]); + EXPECT_EQ(file->write_cols, expected_write_cols[i]); + } + } + private: std::shared_ptr pool_; std::unique_ptr dir_; + std::unique_ptr blob_dir_; + int blob_file_counter_ = 0; arrow::FieldVector fields_ = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("f1"), arrow::field("f2", arrow::utf8())}; }; std::vector GetTestValuesForBlobTableInteTest() { - std::vector values = {"parquet"}; + std::vector values; + values.emplace_back("parquet"); #ifdef PAIMON_ENABLE_ORC values.emplace_back("orc"); #endif @@ -260,142 +535,297 @@ INSTANTIATE_TEST_SUITE_P(FileFormat, BlobTableInteTest, ::testing::ValuesIn(GetTestValuesForBlobTableInteTest())); TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorTrue) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; - - auto generate_blob_array = [&](const std::vector>& blob_descriptors) - -> std::shared_ptr { - arrow::StructBuilder struct_builder( - arrow::struct_(fields), arrow::default_memory_pool(), - {std::make_shared(), std::make_shared(), - std::make_shared()}); - auto string_builder = static_cast(struct_builder.field_builder(0)); - auto int_builder = static_cast(struct_builder.field_builder(1)); - auto binary_builder = - static_cast(struct_builder.field_builder(2)); - for (size_t i = 0; i < blob_descriptors.size(); ++i) { - EXPECT_TRUE(struct_builder.Append().ok()); - EXPECT_TRUE(string_builder->Append("str_" + std::to_string(i)).ok()); - if (i % 3 == 0) { - // test null - EXPECT_TRUE(int_builder->AppendNull().ok()); - } else { - EXPECT_TRUE(int_builder->Append(i).ok()); - } - EXPECT_TRUE( - binary_builder->Append(blob_descriptors[i]->data(), blob_descriptors[i]->size()) - .ok()); - } - std::shared_ptr array; - EXPECT_TRUE(struct_builder.Finish(&array).ok()); - return array; - }; - - // prepare data - std::vector> expected_blob_descriptors; - std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; - ASSERT_OK_AND_ASSIGN(auto blob1, Blob::FromPath(file1)); - expected_blob_descriptors.emplace_back(blob1->ToDescriptor(pool_)); - - std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; - ASSERT_OK_AND_ASSIGN(auto blob2, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); - expected_blob_descriptors.emplace_back(blob2->ToDescriptor(pool_)); - ASSERT_OK_AND_ASSIGN(auto blob3, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); - expected_blob_descriptors.emplace_back(blob3->ToDescriptor(pool_)); - ASSERT_OK_AND_ASSIGN(auto blob4, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); - expected_blob_descriptors.emplace_back(blob4->ToDescriptor(pool_)); - - auto array = generate_blob_array(expected_blob_descriptors); - ::ArrowArray arrow_array; - ASSERT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); - RecordBatchBuilder batch_builder(&arrow_array); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.Finish()); + // prepare data: input uses plain raw blob bytes for readability + std::string raw_json = R"([ + ["str_0", null, "hello_blob_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"], + ["str_3", null, "blob_data_3"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + // write descriptor array + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto schema_with_row_kind = arrow::schema(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null], - [0, "str_1", 1], - [0, "str_2", 2], - [0, "str_3", null] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResultForBlobTable( - schema_with_row_kind, data_splits, expected_data, - expected_blob_descriptors)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // read result contains descriptors pointing to paimon internal blob files + // resolve descriptors back to raw bytes, then prepend _VALUE_KIND and compare + ASSERT_OK_AND_ASSIGN(auto result, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); } TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; - - std::string data = R"([ + std::string data_json = R"([ ["str_0", null, "apple"], ["str_1", 1, "banana"], ["str_2", 2, "cat"], ["str_3", null, "dog"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // BLOB_AS_DESCRIPTOR=false: blob data is stored inline, read result should match input + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); +} + +TEST_P(BlobTableInteTest, TestWriteNullOnMissingFile) { + // blob-write-null-on-missing-file=true: a descriptor whose file is gone at write time + // becomes a NULL blob element, while the row itself is still written and merged with + // the data file on read. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + + // Remove the file behind row 1's descriptor so the write observes a missing file. + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Both the main data file and the .blob file keep the full row count; the read path + // merges them back into rows where row 1's blob is NULL. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestMissingFileFailsWriteWhenWriteNullDisabled) { + // Without blob-write-null-on-missing-file, a missing descriptor file fails the write. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {desc_array}), + "not exists"); +} + +TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) { + // blob-write-null-on-fetch-failure=true: a descriptor whose data cannot be fetched for a + // reason other than a missing file (here: offset beyond the end of the file) becomes a + // NULL blob element, while the row itself is still written and merged with the data file + // on read. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + + // Point row 1's descriptor at an offset beyond the end of its file so the write observes + // a fetch failure that is not a missing file. + ASSERT_OK_AND_ASSIGN(desc_array, CorruptDescriptorOffset(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Both the main data file and the .blob file keep the full row count; the read path + // merges them back into rows where row 1's blob is NULL. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureCoversMissingFile) { + // The existence check runs only under blob-write-null-on-missing-file, so with only + // blob-write-null-on-fetch-failure a missing file is converted as an ordinary fetch failure. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null, "apple"], - [0, "str_1", 1, "banana"], - [0, "str_2", 2, "cat"], - [0, "str_3", null, "dog"] + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); } TEST_P(BlobTableInteTest, TestBasic) { @@ -431,20 +861,74 @@ TEST_P(BlobTableInteTest, TestBasic) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, - SpecialFields::RowId().field_, fields_[2]}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, + SpecialFields::RowId().field_, fields_[2]}), + R"([ ["new_blob", 1, 2, 0, "c"] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, + expected_row_tracking_array)); +} + +TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) { + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Simulate the post-compaction layout: one normal file bridges blob files across schema ids. + std::vector write_cols0 = {"f0", "f2"}; + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0], fields_[2]}), R"([ + [1, "a"], + [2, "b"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, write_cols0, {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + std::vector blob_write_cols = {"f1"}; + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ + ["c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {}, blob_write_cols, {src_array1})); + SetFirstRowId(0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto f3 = arrow::field("f3", arrow::int64()); + ASSERT_OK(WriteNextSchema(table_path, + {DataField(0, fields_[0]), DataField(1, fields_[1]), + DataField(2, fields_[2]), DataField(3, f3)}, + /*highest_field_id=*/3, options)); + + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ + ["d"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, blob_write_cols, {src_array2})); + SetFirstRowId(1, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ + [1, "c", "a"], + [2, "d", "b"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields_)->field_names(), expected_array)); } TEST_P(BlobTableInteTest, TestMultipleAppends) { @@ -536,17 +1020,16 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, "a", "b", 0, 1], [1, "a", "b", 1, 1], [1, "a", "b", 2, 1], @@ -560,57 +1043,705 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) { [1, "a", "b", 10, 2], [2, "c", "d", 11, 4] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } -TEST_P(BlobTableInteTest, TestOnlySomeColumns) { - CreateTable(); +TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - auto schema = arrow::schema(fields_); + auto schema = arrow::schema(fields); - // write field: f0 - std::vector write_cols0 = {"f0"}; + // Initial full-row write assigns row ids 0 and 1. auto src_array0 = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0]}), R"([ - [1] + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "old_blob_0"], + [2, "b", "old_blob_1"] ])") .ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, {}, write_cols0, {src_array0})); - ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); - // write field: f1 - std::vector write_cols1 = {"f1"}; + // Update only b0 and align it with the existing row ids. + std::vector blob_write_cols = {"b0"}; auto src_array1 = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ - ["a"] + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2]}), R"([ + ["new_blob_0"], + ["new_blob_1"] ])") .ValueOrDie()); - ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, write_cols1, {src_array1}), - "Can't infer struct array length with 0 child arrays"); -} + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, + WriteArray(table_path, {}, blob_write_cols, {src_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); -TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) { - CreateTable(); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - auto schema = arrow::schema(fields_); + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "new_blob_0"], + [2, "b", "new_blob_1"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - // First commit, firstRowId = 0 - // write field: f0, f1 - std::vector write_cols1 = {"f0", "f1"}; - auto src_array1 = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0], fields_[1]}), R"([ - [1, "a"] + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", "new_blob_0", 0], + [2, "b", "new_blob_1", 1] ])") .ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, write_cols1, {src_array1})); - SetFirstRowId(0, commit_msgs1); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); +} - // write field: f2 - std::vector write_cols2 = {"f2"}; +/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row +/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob. +std::shared_ptr MakeBlobUpdateArray( + const std::shared_ptr& blob_field, + const std::vector>& rows) { + auto struct_type = arrow::struct_({blob_field}); + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string_view sentinel = BlobDefs::PlaceholderSentinelView(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return std::dynamic_pointer_cast(array); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { + // the blob column is updated to null below, so it must be nullable + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Initial full-row write assigns row ids 0-2. + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "blob_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // Partial update: only row 1 gets a new blob, the untouched rows are written as + // placeholder entries and must fall back to the previous blob file when read. + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // updating a blob to null is not a placeholder: the null must win over older layers + auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, {"b0"}, {null_update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2)); + + // row ids still come from the data files and stay aligned with the fallback result + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", null, 0], + [2, "b", "updated_b", 1], + [3, "c", "blob_c", 2] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); + + // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and + // must still resolve to the same bytes + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(desc_result.chunked_array); + auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie(); + auto desc_struct = std::dynamic_pointer_cast(desc_concat); + ASSERT_TRUE(desc_struct); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "blob_3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // second layer updates row 0 within rows [0, 1] + auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // third layer updates row 3 within rows [2, 3]; each layer only partially covers the + // range, the uncovered parts behave as placeholders + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "update1_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "update2_3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // row-range pushdown where the selected rows fall in one layer's file and in another + // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second + auto expected_middle = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, "b", "blob_1"], + [3, "c", "blob_2"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle, + /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)})); + + // per-row reads resolve every row independently + const std::vector expected_rows = { + R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])", + R"([[4, "d", "update2_3"]])"}; + for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i]) + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups + // (with a single base write, as the C++ write path assigns one sequence per commit): + // row id: 0 1 2 3 4 5 6 7 8 9 + // seq1: [b0 b1 b2 b3 b4 b5 b6 b7 b8 b9] + // seq2: [u20 P P u23 u24] . . . . . + // seq3: . . . . . [P u46 P u48 P] + // seq4: [P u61 P P P P P P P u69] + // result: u20 u61 b2 u23 u24 b5 u46 b7 u48 u69 + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto update_array3 = MakeBlobUpdateArray( + fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + const std::vector expected_blobs = {"u20", "u61", "blob_2", "u23", "u24", + "blob_5", "u46", "blob_7", "u48", "u69"}; + auto expected_row = [&](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] + + "\""; + }; + auto expected_full = PrepareBulkData(10, expected_row, fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); + + // every row resolves independently under row-range pushdown + for (int32_t i = 0; i < 10; i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[" + expected_row(i) + "]]") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // partial update touching rows 1 and 9 only + auto update_array = MakeBlobUpdateArray( + fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // push down row ranges hitting updated and untouched rows + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "name_1", "update_1"], + [5, "name_5", "blob_5"], + [9, "name_9", "update_9"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)})); + + // full read still resolves every row + auto expected_full = PrepareBulkData( + 10, + [](int32_t i) { + std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\"" + : "\"blob_" + std::to_string(i) + "\""; + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob; + }, + fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); +} + +/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id +/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). The table needs a +/// normal column to be creatable, but only the blob column is ever written, so the split +/// contains only blob files and the blob bunch itself carries the row-tracking fields: the +/// fallback reader must keep _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the +/// layer that resolved it, without ever exposing the internal placeholder sentinel. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // old layer: rows 0-9 all real + std::vector> base_rows; + for (int32_t i = 0; i < 10; i++) { + base_rows.emplace_back("b" + std::to_string(i)); + } + auto base_array = MakeBlobUpdateArray(fields[1], base_rows); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // new layer: first_row_id=2, row_count=2, covering only rows 2-3 + auto update_array = MakeBlobUpdateArray(fields[1], {"u2", "u3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 10); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + int64_t old_layer_seq = seq_col->Value(0); + int64_t new_layer_seq = seq_col->Value(2); + ASSERT_GT(new_layer_seq, old_layer_seq); + for (int64_t i = 0; i < 10; i++) { + ASSERT_FALSE(blob_col->IsNull(i)); + std::string expected_blob = + (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i); + // the leading and trailing gaps never expose the internal placeholder sentinel + ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i; + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq) + << "row " << i; + } + + // row-range pushdown with the row-tracking projection: the selection removes rows inside + // the blob files, so batch positions must still map back to the right file row indexes + ASSERT_OK_AND_ASSIGN(auto range_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"}, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 2), Range(8, 8)})); + ASSERT_TRUE(range_read.chunked_array); + auto range_concat = arrow::Concatenate(range_read.chunked_array->chunks()).ValueOrDie(); + auto range_struct = std::dynamic_pointer_cast(range_concat); + ASSERT_TRUE(range_struct); + ASSERT_EQ(range_struct->length(), 3); + auto range_blob_col = + std::dynamic_pointer_cast(range_struct->GetFieldByName("b0")); + auto range_row_id_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::RowId().Name())); + auto range_seq_col = std::dynamic_pointer_cast( + range_struct->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(range_blob_col && range_row_id_col && range_seq_col); + const std::vector expected_row_ids = {1, 2, 8}; + const std::vector expected_blobs = {"b1", "u2", "b8"}; + for (int64_t i = 0; i < 3; i++) { + ASSERT_EQ(range_blob_col->GetString(i), expected_blobs[i]) << "row " << i; + ASSERT_EQ(range_row_id_col->Value(i), expected_row_ids[i]) << "row " << i; + ASSERT_EQ(range_seq_col->Value(i), expected_row_ids[i] == 2 ? new_layer_seq : old_layer_seq) + << "row " << i; + } +} + +/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its +/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. Only the blob column is ever written, so +/// the blob bunch itself carries the row-tracking fields. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // the base layer itself holds a placeholder at row 1, so after the second layer also + // leaves it untouched, row 1 is a placeholder in every layer + auto base_array = MakeBlobUpdateArray(fields[1], {"blob_a", "PH", "blob_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array = MakeBlobUpdateArray(fields[1], {"PH", "PH", "update_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 3); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + + ASSERT_EQ(blob_col->GetString(0), "blob_a"); + ASSERT_EQ(blob_col->GetString(2), "update_c"); + // the all-placeholder row: null blob, row id kept, sequence number -1 + ASSERT_TRUE(blob_col->IsNull(1)); + ASSERT_EQ(row_id_col->Value(1), 1); + ASSERT_EQ(seq_col->Value(1), -1); + for (int64_t i : {static_cast(0), static_cast(2)}) { + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_GE(seq_col->Value(i), 0); + } + ASSERT_GT(seq_col->Value(2), seq_col->Value(0)); +} + +/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly +/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2 +/// entry) and read back unchanged. The sentinel is reserved only inside the data-evolution +/// partial-update channels (see BlobDefs::kPlaceholderSentinel). +TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // full-row write: row 0's blob is exactly the sentinel bytes + auto struct_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(), std::make_shared(), + std::make_shared()}); + auto f0_builder = static_cast(struct_builder.field_builder(0)); + auto f1_builder = static_cast(struct_builder.field_builder(1)); + auto b0_builder = static_cast(struct_builder.field_builder(2)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(1).ok()); + ASSERT_TRUE(f1_builder->Append("a").ok()); + ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok()); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(2).ok()); + ASSERT_TRUE(f1_builder->Append("b").ok()); + ASSERT_TRUE(b0_builder->Append("normal", 6).ok()); + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_struct = std::dynamic_pointer_cast(src_array); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_struct})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // read back unchanged: the sentinel-equal bytes are a normal value + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct)); + + // a value that merely starts with the sentinel bytes stays a normal value even through a + // partial-update fallback (placeholders are identified by exact equality only) + std::string prefixed_bytes = sentinel_bytes + "suffix"; + auto update_array = MakeBlobUpdateArray(fields[2], {prefixed_bytes, "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_result = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_result); + ASSERT_EQ(struct_result->length(), 2); + auto blob_col = + std::dynamic_pointer_cast(struct_result->GetFieldByName("b0")); + ASSERT_TRUE(blob_col); + ASSERT_FALSE(blob_col->IsNull(0)); + ASSERT_EQ(blob_col->GetString(0), prefixed_bytes); + ASSERT_EQ(blob_col->GetString(1), "updated_b"); +} + +TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) { + // Pins the accepted collision of the byte-identified placeholder protocol (see + // BlobDefs::kPlaceholderSentinel): the fallback merge byte-compares every layer, so a user + // blob equal to the sentinel bytes that a later partial update leaves untouched reads as a + // placeholder in every layer and degrades to a null blob instead of falling back. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // the full-row write stores row 0's sentinel-equal bytes verbatim (the write channel is off) + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // the partial update leaves row 0 untouched, so its base value collides with the + // placeholder markers of the newer layer + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // row 0's non-blob fields survive (they come from the data file); only the blob degrades + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); +} + +TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) { + // blob.internal.* options are reserved for the data-evolution write and read paths. Set in + // the user table options they must be ignored: were the write key honored, this full-row + // write would persist the sentinel-equal user value as a placeholder entry that no older + // layer can resolve, making the table unreadable. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {BlobDefs::kWritePlaceholderKey, "true"}, + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + std::string src_json = + std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])"; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {src_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // the sentinel-equal value round-trips verbatim: the user-supplied keys were stripped + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_array)); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto blob_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2]}), R"([ + ["blob_0"] + ])") + .ValueOrDie()); + std::vector blob_write_cols = {"b0"}; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, blob_write_cols, {blob_array})); + + ASSERT_EQ(commit_msgs.size(), 1); + auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]); + ASSERT_TRUE(commit_msg); + ASSERT_EQ(commit_msg->data_increment_.new_files_.size(), 1); + const auto& blob_file = commit_msg->data_increment_.new_files_[0]; + ASSERT_TRUE(BlobUtils::IsBlobFile(blob_file->file_name)); + ASSERT_FALSE(blob_file->first_row_id.has_value()); + + ASSERT_NOK_WITH_MSG(Commit(table_path, commit_msgs), "blobStart 0 should be less than start 0"); +} + +TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + + // First commit, firstRowId = 0 + // write field: f0, f1 + std::vector write_cols1 = {"f0", "f1"}; + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0], fields_[1]}), R"([ + [1, "a"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, write_cols1, {src_array1})); + SetFirstRowId(0, commit_msgs1); + + // write field: f2 + std::vector write_cols2 = {"f2"}; auto src_array2 = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[2]}), R"([ ["b"] @@ -656,25 +1787,23 @@ TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, "a", "b", 0, 1], [2, "c", "d", 1, 3] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(BlobTableInteTest, TestMoreDataWithDataEvolution) { @@ -811,21 +1940,19 @@ TEST_P(BlobTableInteTest, TestExternalPath) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ ["a", 10, "b", 0, 2], ["c", 20, "d", 1, 2] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(BlobTableInteTest, TestPartitionWithPredicate) { @@ -1053,10 +2180,9 @@ TEST_P(BlobTableInteTest, TestIOException) { .ValueOrDie()); auto commit_msgs1_result = WriteArray(table_path, {}, write_cols1, {src_array1}); CHECK_HOOK_STATUS(commit_msgs1_result.status(), i); - SetFirstRowId(/*reset_first_row_id=*/0, - const_cast>&>( - commit_msgs1_result.value())); - CHECK_HOOK_STATUS(Commit(table_path, commit_msgs1_result.value()), i); + auto commit_msgs1 = std::move(commit_msgs1_result).value(); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + CHECK_HOOK_STATUS(Commit(table_path, commit_msgs1), i); write_run_complete = true; break; } @@ -1167,7 +2293,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { DataField(3, arrow::field("f1", arrow::utf8())), DataField(4, arrow::field("f2", arrow::decimal128(6, 3))), DataField(5, arrow::field("f0", arrow::boolean())), - DataField(8, BlobUtils::ToArrowField("f5")), + DataField(8, BlobUtils::ToArrowField("blob")), DataField(9, arrow::field("f6", arrow::int32())), SpecialFields::RowId(), SpecialFields::SequenceNumber()}; @@ -1179,7 +2305,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { // only read blob column auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({BlobUtils::ToArrowField("f5")}), R"([ + arrow::struct_({BlobUtils::ToArrowField("blob")}), R"([ ["Lily"], ["Alice"], ["Bob"], @@ -1192,7 +2318,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { ["Elderberry"] ])") .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f5"}, expected_array)); + ASSERT_OK(ScanAndRead(table_path, {"blob"}, expected_array)); } { auto expected_array = std::dynamic_pointer_cast( @@ -1291,7 +2417,6 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(/*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -1377,7 +2502,7 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { /*row_ranges=*/{Range(1l, 1l), Range(5l, 5l)})); } { - // test ont read blob field + // test not read blob field auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ ["aaa1"], @@ -1391,107 +2516,60 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { } TEST_P(BlobTableInteTest, TestAppendTableWriteWithMultipleBlobFields) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob1", true), BlobUtils::ToArrowField("blob2", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; - - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - std::string data = R"([ + std::string data_json = R"([ ["str_0", null, "apple", "red"], ["str_1", 1, "banana", "yellow"], ["str_2", 2, "cat", "black"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - ASSERT_EQ(commit_msgs.size(), 1); - - ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); - ASSERT_TRUE(snapshot); - ASSERT_EQ(1, snapshot.value().Id()); - ASSERT_EQ(3, snapshot.value().NextRowId().value()); - - // Scan and read: verify all fields including multiple blob fields - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null, "apple", "red"], - [0, "str_1", 1, "banana", "yellow"], - [0, "str_2", 2, "cat", "black"] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}}; - - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); // Write: row 0 non-null blob, row 1 null blob, row 2 non-null blob - std::string data = R"([ + std::string data_json = R"([ [1, "hello"], [2, null], [3, "world"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, - /*expected_commit_messages=*/std::nullopt)); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); - // Read and verify - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, 1, "hello"], - [0, 2, null], - [0, 3, "world"] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { @@ -1567,4 +2645,1448 @@ TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { } } +TEST_P(BlobTableInteTest, TestBlobDescriptorField) { + // Two blob fields configured via BLOB_DESCRIPTOR_FIELD and stored inline as descriptors. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes for readability + std::string raw_json = R"([ + [1, "image_data_0", "video_data_0"], + [2, "image_data_1", "video_data_1"], + [3, "image_data_2", "video_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0", "b1"})); + + // write descriptor array + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: all blob fields are inline descriptors, so write_cols is unset. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, + /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, + /*expected_first_row_ids=*/{0}, + /*expected_write_cols=*/{std::nullopt}); + + // Read and resolve descriptors back to raw bytes + std::map read_options = {}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // Descriptor bytes should be unchanged (inline, not repacked) + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_TRUE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { + // 4 blob fields: b0,b1 are inline descriptors; b2,b3 are regular blob fields written to + // .blob files. + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes: + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, + ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); + + // write: b0,b1 as descriptor bytes; b2,b3 as raw bytes (paimon writes them to .blob files) + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: b2,b3 go to .blob files, "f0", "b0", "b1" go to main files. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, + /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, + /*expected_first_row_ids=*/{0, 0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, + std::vector{"b3"}}); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // b0,b1 inline descriptor (not repacked), should match input + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_TRUE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); + + // Resolve b0,b1 descriptors back to raw bytes, then compare full struct + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { + // Multiple write+commit rounds with a shuffled read schema: b3, b2, b1, b0, f0. + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // --- First write+commit --- + std::string raw_json_1 = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ] + ])"; + auto raw_array_1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_1).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_1, ConvertRawBlobToDescriptor(raw_array_1, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + WriteArray(table_path, {}, schema->field_names(), {desc_array_1})); + ASSERT_OK(Commit(table_path, commit_msgs_1)); + + // --- Second write+commit --- + std::string raw_json_2 = R"([ + [3, "img_2", "vid_2", "raw_2_2", "raw_3_2"], + [4, null, "vid_3", "raw_2_3", "raw_3_3"] + ])"; + auto raw_array_2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_2).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_2, ConvertRawBlobToDescriptor(raw_array_2, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + WriteArray(table_path, {}, schema->field_names(), {desc_array_2})); + ASSERT_OK(Commit(table_path, commit_msgs_2)); + + // --- Third write+commit --- + std::string raw_json_3 = R"([ + [5, "img_4", null, "raw_2_4", null ], + [6, "img_5", "vid_5", null, "raw_3_5"] + ])"; + auto raw_array_3 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_3).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_3, ConvertRawBlobToDescriptor(raw_array_3, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_3, + WriteArray(table_path, {}, schema->field_names(), {desc_array_3})); + ASSERT_OK(Commit(table_path, commit_msgs_3)); + + // test read + { + // --- Read with shuffled schema: b3, b2, b1, b0, f0 --- + std::vector shuffled_read_schema = {"b3", "b2", "b1", "b0", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, shuffled_read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Build expected array in shuffled order from all 3 batches + arrow::FieldVector shuffled_fields = { + BlobUtils::ToArrowField("b3", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b0", true), + arrow::field("f0", arrow::int32())}; + std::string expected_json = R"([ + ["raw_3_0", "raw_2_0", null, "img_0", 1], + [null, "raw_2_1", "vid_1", "img_1", 2], + ["raw_3_2", "raw_2_2", "vid_2", "img_2", 3], + ["raw_3_3", "raw_2_3", "vid_3", null, 4], + [null, "raw_2_4", null, "img_4", 5], + ["raw_3_5", null, "vid_5", "img_5", 6] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(shuffled_fields), + expected_json) + .ValueOrDie()); + + // Resolve descriptors (b0, b1 are descriptor fields) back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + } + { + // test scan and read with GlobalIndexResult + std::vector shuffled_read_schema = {"b3", "b2", "b1", "b0", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path, /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 3), Range(5, 5)})); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, shuffled_read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Build expected array in shuffled order from all 3 batches + arrow::FieldVector shuffled_fields = { + BlobUtils::ToArrowField("b3", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b0", true), + arrow::field("f0", arrow::int32())}; + std::string expected_json = R"([ + [null, "raw_2_1", "vid_1", "img_1", 2], + ["raw_3_2", "raw_2_2", "vid_2", "img_2", 3], + ["raw_3_3", "raw_2_3", "vid_3", null, 4], + ["raw_3_5", null, "vid_5", "img_5", 6] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(shuffled_fields), + expected_json) + .ValueOrDie()); + + // Resolve descriptors (b0, b1 are descriptor fields) back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + } +} + +// The shared-shredding map is read from one main data file while the blob payload is read from a +// separate blob file with the same row-id range. +TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) { + if (GetParam() == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto map_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["a", 20]]], + [3, null] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto map_msgs, WriteArray(table_path, {}, {"id", "tags"}, {map_array})); + ASSERT_OK(Commit(table_path, map_msgs)); + + auto blob_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), + R"([ + [1, "payload-1"], + [2, "payload-2"], + [3, "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto blob_msgs, + WriteArray(table_path, {}, {"id", "payload"}, {blob_array})); + SetFirstRowId(0, blob_msgs); + ASSERT_OK(Commit(table_path, blob_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], "payload-1"], + [2, [["a", 20]], "payload-2"], + [3, null, "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +// Two independent shared-shredding map columns are written into different main files. +TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) { + if (GetParam() == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("f0", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::utf8())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.f0.map.storage-layout", "shared-shredding"}, + {"fields.f0.map.shared-shredding.max-columns", "1"}, + {"fields.f1.map.storage-layout", "shared-shredding"}, + {"fields.f1.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto f0_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["a", 20]]], + [3, null] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto f0_msgs, WriteArray(table_path, {}, {"id", "f0"}, {f0_array})); + ASSERT_OK(Commit(table_path, f0_msgs)); + + auto f1_blob_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2], fields[3]}), + R"([ + [[["b", "red"], ["y", "blue"]], "payload-1"], + [[["b", "green"]], "payload-2"], + [[], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto f1_blob_msgs, + WriteArray(table_path, {}, {"f1", "payload"}, {f1_blob_array})); + SetFirstRowId(0, f1_blob_msgs); + ASSERT_OK(Commit(table_path, f1_blob_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], [["b", "red"], ["y", "blue"]], "payload-1"], + [2, [["a", 20]], [["b", "green"]], "payload-2"], + [3, null, [], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +// A newer partial data file rewrites only the shared-shredding map for the same row-id range. +TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) { + if (GetParam() == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto old_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], "payload-1"], + [2, [["a", 20]], "payload-2"], + [3, [["a", 30]], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto old_msgs, + WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {old_array})); + ASSERT_OK(Commit(table_path, old_msgs)); + + auto new_map_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}), + R"([ + [[["a", 100], ["z", 101]]], + [null], + [[]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto new_map_msgs, WriteArray(table_path, {}, {"tags"}, {new_map_array})); + SetFirstRowId(0, new_map_msgs); + ASSERT_OK(Commit(table_path, new_map_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 100], ["z", 101]], "payload-1"], + [2, null, "payload-2"], + [3, [], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution) { + if (GetParam() != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + BlobUtils::ToArrowField("payload"), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "default"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options_v0); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto array_v0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]], "payload-1"], + [2, [["a", "red"], ["z", "green"]], "payload-2"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto msgs_v0, WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {array_v0})); + ASSERT_OK(Commit(table_path, msgs_v0)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema( + table_path, {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])}, + /*highest_field_id=*/2, options_v1)); + + auto array_v1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [3, [["a", "red"], ["z", "yellow"]], "payload-3"], + [4, [["a", "red"]], "payload-4"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto msgs_v1, WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {array_v1})); + ASSERT_OK(Commit(table_path, msgs_v1)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]], "payload-1"], + [2, [["a", "red"], ["z", "green"]], "payload-2"], + [3, [["a", "red"], ["z", "yellow"]], "payload-3"], + [4, [["a", "red"]], "payload-4"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { + // Test DataEvolution (split-column write) combined with blob descriptor fields. + // Schema: f0(int32), b0/b1(blob descriptor inline), b2/b3(blob). + // Commit 1: file A writes (f0, b2, b3) + // Commit 2: file B writes (f0, b0, b1) with SetFirstRowId(0) + // -> merges with commit 1 + // Commit 3: file A writes (f0, b0, b1, b3) + // Commit 4: file B writes (b0, b1, b3) with SetFirstRowId(3) + // -> merges with commit 3 + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // --- Commit 1: file A (f0, b2, b3), Commit 2: file B (f0, b0, b1) SetFirstRowId(0) --- + std::string file_a1_json = R"([ + [1, "raw_2_0", "raw_3_0"], + [2, "raw_2_1", null ], + [3, null, "raw_3_2"] + ])"; + arrow::FieldVector file_a1_fields = {fields[0], fields[3], fields[4]}; + auto file_a1_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_a1_fields), file_a1_json) + .ValueOrDie()); + + std::string file_b1_json = R"([ + [1, "img_0", "vid_0"], + [2, "img_1", null ], + [3, "img_2", "vid_2"] + ])"; + arrow::FieldVector file_b1_fields = {fields[0], fields[1], fields[2]}; + auto file_b1_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b1_fields), file_b1_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_b1_desc, + ConvertRawBlobToDescriptor(file_b1_array, {"b0", "b1"})); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_a1, + WriteArray(table_path, {}, {"f0", "b2", "b3"}, {file_a1_array})); + ASSERT_OK(Commit(table_path, commit_msgs_a1)); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_b1, + WriteArray(table_path, {}, {"f0", "b0", "b1"}, {file_b1_desc})); + SetFirstRowId(0, commit_msgs_b1); + ASSERT_OK(Commit(table_path, commit_msgs_b1)); + + // --- Commit 3: file A (f0, b0, b1, b3), Commit 4: file B (b0, b1, b3) SetFirstRowId(3) --- + // Duplicate cols b0, b1, b3: file B (commit 4, newer) takes precedence. + std::string file_a2_json = R"([ + [4, "img_3_old", "vid_3_old", "raw_3_3_old"], + [5, null, "vid_4_old", "raw_3_4_old"], + [6, "img_5_old", null, null ] + ])"; + arrow::FieldVector file_a2_fields = {fields[0], fields[1], fields[2], fields[4]}; + auto file_a2_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_a2_fields), file_a2_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_a2_desc, + ConvertRawBlobToDescriptor(file_a2_array, {"b0", "b1"})); + + std::string file_b2_json = R"([ + ["img_3", "vid_3", "raw_3_3"], + [null, "vid_4", "raw_3_4"], + ["img_5", null, null ] + ])"; + arrow::FieldVector file_b2_fields = {fields[1], fields[2], fields[4]}; + auto file_b2_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b2_fields), file_b2_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_b2_desc, + ConvertRawBlobToDescriptor(file_b2_array, {"b0", "b1"})); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_a2, + WriteArray(table_path, {}, {"f0", "b0", "b1", "b3"}, {file_a2_desc})); + ASSERT_OK(Commit(table_path, commit_msgs_a2)); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_b2, + WriteArray(table_path, {}, {"b0", "b1", "b3"}, {file_b2_desc})); + SetFirstRowId(3, commit_msgs_b2); + ASSERT_OK(Commit(table_path, commit_msgs_b2)); + + // --- Read all data with full schema --- + std::vector read_schema = {"f0", "b0", "b1", "b2", "b3"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 6); + + // Expected: round1 all columns present; round2 b2=null, b0/b1/b3 from file B (newer) + std::string expected_json = R"([ + [1, "img_0", "vid_0", "raw_2_0", "raw_3_0"], + [2, "img_1", null, "raw_2_1", null ], + [3, "img_2", "vid_2", null, "raw_3_2" ], + [4, "img_3", "vid_3", null, "raw_3_3" ], + [5, null, "vid_4", null, "raw_3_4" ], + [6, "img_5", null, null, null ] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + + // Resolve descriptors back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->type()->Equals(expected_with_rk->type())); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { + // Similar to TestBlobDescriptorField but writes raw bytes directly without converting to + // descriptor first. Descriptor fields reject values without the descriptor magic header. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write raw bytes directly (no ConvertRawBlobToDescriptor) + std::string raw_json = R"([ + [1, "image_data_0", "video_data_0"], + [2, "image_data_1", "video_data_1"], + [3, "image_data_2", "video_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {raw_array}), + "BLOB inline field b0 require values to be set as corresponding type."); +} + +TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + const std::string upstream_db_name = "append_table_with_multi_blob"; + const std::string upstream_table_name = "append_table_with_multi_blob"; + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + upstream_db_name + + ".db/" + upstream_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // src array + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int32_t i = 0; i < 8; ++i) { + if (i < 6) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/6, + /*row_id=*/static_cast(i)); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE(view_builder + .Append(reinterpret_cast(serialized->data()), + serialized->size()) + .ok()); + } else { + ASSERT_TRUE(view_builder.AppendNull().ok()); + } + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + // write & commit + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + std::string padding_b(2048, 'b'); + std::string padding_d(2048, 'd'); + std::string padding_e(2048, 'e'); + std::string padding_f(2048, 'f'); + + // scan & read + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + // clang-format off + std::string expected_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); + + // Sub-case 1: scan with row_ranges. + { + ASSERT_OK_AND_ASSIGN(auto range_plan, ScanTable(table_path, /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 3), Range(5, 5)})); + ASSERT_OK_AND_ASSIGN(auto range_result, ReadTable(table_path, schema->field_names(), + range_plan, /*predicate=*/nullptr)); + ASSERT_TRUE(range_result.chunked_array); + auto range_concat = arrow::Concatenate(range_result.chunked_array->chunks()).ValueOrDie(); + auto range_struct = std::dynamic_pointer_cast(range_concat); + ASSERT_EQ(range_struct->length(), 4); + ASSERT_OK_AND_ASSIGN(auto range_resolved, + ConvertDescriptorToRawBlob(range_struct, {"view"})); + + // clang-format off + std::string range_json = R"([ +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[105, ")" + padding_f + R"("] +])"; + // clang-format on + auto range_expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), range_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto range_expected_with_rk, + PrependRowKindColumn(range_expected_struct)); + ASSERT_TRUE(range_resolved->Equals(range_expected_with_rk)) + << "range_resolved:" << range_resolved->ToString() << std::endl + << "expected:" << range_expected_with_rk->ToString(); + } + + // Sub-case 2: scan with predicate (f0 > 102), data evolution split read will ignore format push + // down + { + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(102)); + ASSERT_OK_AND_ASSIGN(auto pred_plan, ScanTable(table_path, predicate, /*row_ranges=*/{})); + ASSERT_OK_AND_ASSIGN(auto pred_result, + ReadTable(table_path, schema->field_names(), pred_plan, predicate)); + ASSERT_TRUE(pred_result.chunked_array); + auto pred_concat = arrow::Concatenate(pred_result.chunked_array->chunks()).ValueOrDie(); + auto pred_struct = std::dynamic_pointer_cast(pred_concat); + ASSERT_EQ(pred_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto pred_resolved, ConvertDescriptorToRawBlob(pred_struct, {"view"})); + + // clang-format off + std::string pred_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto pred_expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), pred_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto pred_expected_with_rk, + PrependRowKindColumn(pred_expected_struct)); + ASSERT_TRUE(pred_resolved->Equals(pred_expected_with_rk)) + << "pred_resolved:" << pred_resolved->ToString() << std::endl + << "expected:" << pred_expected_with_rk->ToString(); + } +} + +TEST_P(BlobTableInteTest, TestForwardBlobViewReference) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + // Forward blob view references between two blob-view tables: read the source table with + // resolve dynamically disabled, write the preserved BlobViewStruct bytes into the target + // table, then verify the target still stores the original upstream references and a + // default read resolves them to the actual upstream blob values. + const std::string upstream_db_name = "append_table_with_multi_blob"; + const std::string upstream_table_name = "append_table_with_multi_blob"; + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + upstream_db_name + + ".db/" + upstream_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + + // The source table has no upstream warehouse configured: only a read with resolve + // dynamically disabled can succeed on it. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map source_options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, source_options); + std::string source_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // The target table configures the upstream warehouse for resolving forwarded references. + std::map target_options = source_options; + target_options[Options::BLOB_VIEW_UPSTREAM_WAREHOUSE] = dir_->Str(); + auto schema = arrow::schema(fields); + ::ArrowSchema c_target_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_target_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir_->Str(), {})); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar_forward"), &c_target_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, target_options, + /*ignore_if_exists=*/false)); + std::string target_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar_forward"); + + // src array + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int32_t i = 0; i < 8; ++i) { + if (i < 6) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/6, + /*row_id=*/static_cast(i)); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE(view_builder + .Append(reinterpret_cast(serialized->data()), + serialized->size()) + .ok()); + } else { + ASSERT_TRUE(view_builder.AppendNull().ok()); + } + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + // write & commit into the source table + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(source_table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(source_table_path, commit_msgs)); + + // A default read fails on the missing upstream warehouse: the pass-through below is + // enabled by the dynamic option alone. + ASSERT_OK_AND_ASSIGN(auto source_plan, ScanTable(source_table_path)); + ASSERT_NOK_WITH_MSG( + ReadTable(source_table_path, schema->field_names(), source_plan, /*predicate=*/nullptr), + "BLOB_VIEW_UPSTREAM_WAREHOUSE"); + + ASSERT_OK_AND_ASSIGN( + auto source_result, + ReadTable(source_table_path, schema->field_names(), source_plan, /*predicate=*/nullptr, + {{Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + ASSERT_TRUE(source_result.chunked_array); + auto source_concat = arrow::Concatenate(source_result.chunked_array->chunks()).ValueOrDie(); + auto source_struct = std::dynamic_pointer_cast(source_concat); + ASSERT_EQ(source_struct->length(), 8); + auto forward_f0_array = source_struct->GetFieldByName("f0"); + ASSERT_TRUE(forward_f0_array); + auto forward_view_array = source_struct->GetFieldByName("view"); + ASSERT_TRUE(forward_view_array); + ASSERT_TRUE(forward_view_array->Equals(write_view_array)) + << "source view:" << forward_view_array->ToString() << std::endl + << "written view:" << write_view_array->ToString(); + + // Forward the preserved references into the target blob-view table. + auto forward_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({forward_f0_array, forward_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto forward_commit_msgs, + WriteArray(target_table_path, {}, schema->field_names(), {forward_struct})); + ASSERT_OK(Commit(target_table_path, forward_commit_msgs)); + + // The target table stores the original upstream references byte-identically. + ASSERT_OK_AND_ASSIGN(auto target_plan, ScanTable(target_table_path)); + ASSERT_OK_AND_ASSIGN( + auto raw_target_result, + ReadTable(target_table_path, schema->field_names(), target_plan, /*predicate=*/nullptr, + {{Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + ASSERT_TRUE(raw_target_result.chunked_array); + auto raw_target_concat = + arrow::Concatenate(raw_target_result.chunked_array->chunks()).ValueOrDie(); + auto raw_target_struct = std::dynamic_pointer_cast(raw_target_concat); + ASSERT_EQ(raw_target_struct->length(), 8); + auto raw_target_view_array = raw_target_struct->GetFieldByName("view"); + ASSERT_TRUE(raw_target_view_array); + ASSERT_TRUE(raw_target_view_array->Equals(write_view_array)) + << "target view:" << raw_target_view_array->ToString() << std::endl + << "written view:" << write_view_array->ToString(); + + // A default read of the target table resolves to the actual upstream blob values. + ASSERT_OK_AND_ASSIGN(auto resolved_result, + ReadTable(target_table_path, schema->field_names(), target_plan, + /*predicate=*/nullptr)); + ASSERT_TRUE(resolved_result.chunked_array); + auto resolved_concat = arrow::Concatenate(resolved_result.chunked_array->chunks()).ValueOrDie(); + auto resolved_struct = std::dynamic_pointer_cast(resolved_concat); + ASSERT_EQ(resolved_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(resolved_struct, {"view"})); + + std::string padding_b(2048, 'b'); + std::string padding_d(2048, 'd'); + std::string padding_e(2048, 'e'); + std::string padding_f(2048, 'f'); + // clang-format off + std::string expected_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "resolved:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { + auto file_format = GetParam(); + // Upstream table has two blob descriptor fields. The downstream view references cells from + // both b0 (field_id=1) and b1 (field_id=2). + const std::string upstream_db_name = "upstream_two_blob"; + const std::string upstream_table_name = "upstream_two_blob"; + arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + auto upstream_schema = arrow::schema(upstream_fields); + std::map upstream_options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::FILE_SYSTEM, "local"}}; + + ::ArrowSchema upstream_c_schema; + ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &upstream_c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto upstream_catalog, + Catalog::Create(dir_->Str(), {{Options::FILE_SYSTEM, "local"}})); + ASSERT_OK(upstream_catalog->CreateDatabase(upstream_db_name, {}, /*ignore_if_exists=*/true)); + ASSERT_OK(upstream_catalog->CreateTable( + Identifier(upstream_db_name, upstream_table_name), &upstream_c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, upstream_options, + /*ignore_if_exists=*/false)); + std::string upstream_table_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + + // Write 4 rows of b0/b1 data into the upstream table. + std::string upstream_raw_json = R"([ +[0, "b0_data_0", "b1_data_0"], +[1, "b0_data_1", "b1_data_1"], +[2, "b0_data_2", "b1_data_2"], +[3, "b0_data_3", "b1_data_3"] +])"; + auto upstream_raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(upstream_fields), + upstream_raw_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto upstream_desc_array, + ConvertRawBlobToDescriptor(upstream_raw_array, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN( + auto upstream_commit_msgs, + WriteArray(upstream_table_path, {}, upstream_schema->field_names(), {upstream_desc_array})); + ASSERT_OK(Commit(upstream_table_path, upstream_commit_msgs)); + + // Create the downstream blob-view table that references both b0 and b1 of the upstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Build the view column mixing references to b0 (field_id=1) and b1 (field_id=2). + // - row 0: b0 row 0 -> "b0_data_0" + // - row 1: b1 row 1 -> "b1_data_1" + // - row 2: b0 row 2 -> "b0_data_2" + // - row 3: b1 row 3 -> "b1_data_3" + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + auto append_view = [&](int32_t field_id, int64_t row_id, arrow::LargeBinaryBuilder* builder) { + BlobViewStruct view_struct(upstream_identifier, field_id, row_id); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + builder + ->Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + }; + arrow::LargeBinaryBuilder view_builder; + append_view(/*field_id=*/1, /*row_id=*/0, &view_builder); + append_view(/*field_id=*/2, /*row_id=*/1, &view_builder); + append_view(/*field_id=*/1, /*row_id=*/2, &view_builder); + append_view(/*field_id=*/2, /*row_id=*/3, &view_builder); + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100,101,102,103])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan/read the downstream table and verify the resolved view blobs. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 4); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + std::string expected_json = R"([ +[100, "b0_data_0"], +[101, "b1_data_1"], +[102, "b0_data_2"], +[103, "b1_data_3"] +])"; + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + // Upstream table 1: append_table_with_multi_blob, with two blob fields f5 (field_id=5) and + // f6 (field_id=6). + const std::string multi_blob_db_name = "append_table_with_multi_blob"; + const std::string multi_blob_table_name = "append_table_with_multi_blob"; + { + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + + multi_blob_db_name + ".db/" + multi_blob_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), multi_blob_db_name + ".db/" + multi_blob_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + } + + // Upstream table 2: blob_append_table_alter_table_with_cast_with_data_evolution, with one blob + // field blob (field_id=8). + const std::string alter_db_name = "blob_append_table_alter_table_with_cast_with_data_evolution"; + const std::string alter_table_name = + "blob_append_table_alter_table_with_cast_with_data_evolution"; + { + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + alter_db_name + + ".db/" + alter_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), alter_db_name + ".db/" + alter_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + } + + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view1", true), + BlobUtils::ToArrowField("view2", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view1,view2"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + Identifier multi_blob_identifier(multi_blob_db_name, multi_blob_table_name); + Identifier alter_identifier(alter_db_name, alter_table_name); + + auto append_view = [&](const Identifier& identifier, int32_t field_id, int64_t row_id, + arrow::LargeBinaryBuilder* builder) { + BlobViewStruct view_struct(identifier, field_id, row_id); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + builder + ->Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + }; + + // Build view1 column. References multi_blob.f5 (field_id=5) and f6 (field_id=6). + // Some upstream cells are referenced more than once on purpose. + // - row 0: f5 row 3 -> 'D' * 1024 + // - row 1: f6 row 1 -> 'b' * 2048 + // - row 2: f5 row 3 -> 'D' * 1024 (repeat of row 0) + // - row 3: f6 row 4 -> 'e' * 2048 + // - row 4: f5 row 3 -> 'D' * 1024 (repeat of row 0) + // - row 5: f6 row 1 -> 'b' * 2048 (repeat of row 1) + // - row 6: f5 row 5 -> 'F' * 1024 + // - row 7: f6 row 6 -> 'g' * 2048 + arrow::LargeBinaryBuilder view1_builder; + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/1, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/4, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/1, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/5, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/6, &view1_builder); + std::shared_ptr write_view1_array; + ASSERT_TRUE(view1_builder.Finish(&write_view1_array).ok()); + + // Build view2 column. References alter.blob (field_id=8). + // Some upstream cells are referenced more than once on purpose. + // - row 0: blob row 0 -> "Lily" + // - row 1: blob row 5 -> "Apple" + // - row 2: blob row 0 -> "Lily" (repeat of row 0) + // - row 3: blob row 2 -> "Bob" + // - row 4: blob row 5 -> "Apple" (repeat of row 1) + // - row 5: blob row 9 -> "Elderberry" + // - row 6: blob row 0 -> "Lily" (repeat of row 0) + // - row 7: blob row 3 -> "Cindy" + arrow::LargeBinaryBuilder view2_builder; + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/5, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/2, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/5, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/9, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/3, &view2_builder); + std::shared_ptr write_view2_array; + ASSERT_TRUE(view2_builder.Finish(&write_view2_array).ok()); + + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make( + arrow::ArrayVector({write_f0_array, write_view1_array, write_view2_array}), + std::vector({"f0", "view1", "view2"})) + .ValueOrDie()); + + // write & commit + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Expected blob contents per referenced upstream cell. + std::string blob_f5_row3(1024, 'D'); // multi_blob.f5 row 3 + std::string blob_f5_row5(1024, 'F'); // multi_blob.f5 row 5 + std::string blob_f6_row1(2048, 'b'); // multi_blob.f6 row 1 + std::string blob_f6_row4(2048, 'e'); // multi_blob.f6 row 4 + std::string blob_f6_row6(2048, 'g'); // multi_blob.f6 row 6 + + std::vector read_fields = {"view2", "view1", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, read_fields, plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto result_array, + ConvertDescriptorToRawBlob(read_struct, {"view1", "view2"})); + + // Expected struct follows the requested (shuffled) column order: view2, view1, f0. + arrow::FieldVector expected_fields = {BlobUtils::ToArrowField("view2", true), + BlobUtils::ToArrowField("view1", true), + arrow::field("f0", arrow::int32())}; + // clang-format off + std::string expected_json = R"([ +["Lily", ")" + blob_f5_row3 + R"(", 100], +["Apple", ")" + blob_f6_row1 + R"(", 101], +["Lily", ")" + blob_f5_row3 + R"(", 102], +["Bob", ")" + blob_f6_row4 + R"(", 103], +["Apple", ")" + blob_f5_row3 + R"(", 104], +["Elderberry", ")" + blob_f6_row1 + R"(", 105], +["Lily", ")" + blob_f5_row5 + R"(", 106], +["Cindy", ")" + blob_f6_row6 + R"(", 107] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(expected_fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) { + auto file_format = GetParam(); + auto upstream_dir = UniqueTestDirectory::Create("local"); + const std::string upstream_db_name = "nonexistent_db"; + const std::string upstream_table_name = "nonexistent_table"; + + // Build downstream table that references the non-existent upstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, upstream_dir->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write a single row with a BlobViewStruct pointing to the non-existent upstream table. + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/2, /*row_id=*/0); + auto serialized = view_struct.Serialize(pool_); + arrow::LargeBinaryBuilder view_builder; + ASSERT_TRUE( + view_builder + .Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100])").ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Reading should fail because both paths are absent. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), + "Ambiguous table path"); +} + +TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { + auto file_format = GetParam(); + const std::string upstream_db_name = "fallback_db"; + const std::string upstream_table_name = "fallback_table"; + arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + auto upstream_schema = arrow::schema(upstream_fields); + std::map upstream_options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::FILE_SYSTEM, "local"}}; + + // Create the upstream table at the fallback path: /db/table (no .db). + auto upstream_dir = UniqueTestDirectory::Create("local"); + std::string fallback_table_path = + PathUtil::JoinPath(upstream_dir->Str(), upstream_db_name + "/" + upstream_table_name); + + // Manually create schema at fallback path so it can be read as a valid paimon table. + { + // Use a temporary warehouse with Catalog to build the table data, then copy to fallback. + auto temp_dir = UniqueTestDirectory::Create("local"); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, + Catalog::Create(temp_dir->Str(), {{Options::FILE_SYSTEM, "local"}})); + ASSERT_OK(catalog->CreateDatabase(upstream_db_name, {}, /*ignore_if_exists=*/true)); + ASSERT_OK(catalog->CreateTable(Identifier(upstream_db_name, upstream_table_name), &c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, upstream_options, + /*ignore_if_exists=*/false)); + std::string temp_table_path = + PathUtil::JoinPath(temp_dir->Str(), upstream_db_name + ".db/" + upstream_table_name); + + // Write data to the temp table. + std::string raw_json = R"([[0, "hello"], [1, "world"]])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(upstream_fields), raw_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK_AND_ASSIGN( + auto upstream_commit_msgs, + WriteArray(temp_table_path, {}, upstream_schema->field_names(), {desc_array})); + ASSERT_OK(Commit(temp_table_path, upstream_commit_msgs)); + + // Copy the temp table to the fallback path (without .db). + ASSERT_TRUE(TestUtil::CopyDirectory(temp_table_path, fallback_table_path)); + } + + // Build the downstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, upstream_dir->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write downstream rows referencing the upstream fallback table. + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int64_t row = 0; row < 2; ++row) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/1, /*row_id=*/row); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + view_builder + .Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100, 101])").ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Read and verify + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 2); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + std::string expected_json = R"([[100, "hello"], [101, "world"]])"; + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + std::string table_path = + GetDataDir() + "/" + file_format + "/blob_desc_field.db/blob_desc_field"; + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + auto schema = arrow::schema(fields); + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // After read, b0 and b1 are both descriptor-stored; resolve all back to raw bytes. + // Java-generated descriptors may contain absolute paths from the generation machine. + // Rewrite them to the portable blob directories inside the copied table path. + BlobDescriptorPathRewrite rewrite{table_path, {"raw_blob", "external_blob"}}; + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"}, rewrite)); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + } // namespace paimon::test diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 49d2af3c..536bdc74 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -344,8 +344,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); ASSERT_EQ(30, snapshot_3.Watermark().value()); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); ASSERT_OK_AND_ASSIGN(bool f1_10_bucket_0_exist, @@ -508,8 +508,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(snapshot_exist, commit_impl->snapshot_manager_->SnapshotExists(3)); ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 4c040fcf..f6a2635d 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -7,18 +7,23 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ +#include + #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" @@ -36,9 +41,11 @@ #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +using DataEvolutionTableParam = std::tuple; + // This is a sdk end-to-end test for data evolution class DataEvolutionTableTest : public ::testing::Test, - public ::testing::WithParamInterface { + public ::testing::WithParamInterface { void SetUp() override { dir_ = UniqueTestDirectory::Create("local"); int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); @@ -48,9 +55,10 @@ class DataEvolutionTableTest : public ::testing::Test, dir_.reset(); } - void CreateTable(const std::vector& partition_keys, + void CreateTable(const arrow::FieldVector& fields, + const std::vector& partition_keys, const std::map& options) const { - auto schema = arrow::schema(fields_); + auto schema = arrow::schema(fields); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -61,9 +69,14 @@ class DataEvolutionTableTest : public ::testing::Test, /*ignore_if_exists=*/false)); } + void CreateTable(const std::vector& partition_keys, + const std::map& options) const { + CreateTable(fields_, partition_keys, options); + } + void CreateTable(const std::vector& partition_keys) const { std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -125,6 +138,19 @@ class DataEvolutionTableTest : public ::testing::Test, return file_store_commit->Commit(commit_msgs); } + Status CommitWithRowIdCheckFromSnapshot( + const std::string& table_path, + const std::vector>& commit_msgs, + std::optional row_id_check_from_snapshot) const { + CommitContextBuilder commit_builder(table_path, "commit_user_1"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); + return file_store_commit->Commit(commit_msgs); + } + Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, const std::shared_ptr& expected_array, const std::shared_ptr& predicate = nullptr, @@ -137,7 +163,7 @@ class DataEvolutionTableTest : public ::testing::Test, auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { @@ -149,7 +175,7 @@ class DataEvolutionTableTest : public ::testing::Test, // read auto splits = result_plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); + read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); @@ -203,7 +229,7 @@ class DataEvolutionTableTest : public ::testing::Test, auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); @@ -229,6 +255,26 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } + Result> FinishScanContext(ScanContextBuilder& builder) const { + if (EnableSnapshotLiveManifestCache()) { + if (!snapshot_live_manifest_cache_) { + snapshot_live_manifest_cache_ = + std::make_shared(/*max_weight=*/64 * 1024 * 1024); + } + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(snapshot_live_manifest_cache_); + } + return builder.Finish(); + } + + std::string FileFormat() const { + return std::get<0>(GetParam()); + } + + bool EnableSnapshotLiveManifestCache() const { + return std::get<1>(GetParam()); + } + std::shared_ptr PrepareBulkData( int32_t write_batch_size, std::function data_generator, const arrow::FieldVector& fields) const { @@ -248,6 +294,7 @@ class DataEvolutionTableTest : public ::testing::Test, private: std::unique_ptr dir_; + mutable std::shared_ptr snapshot_live_manifest_cache_; arrow::FieldVector fields_ = { arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), @@ -288,27 +335,65 @@ TEST_P(DataEvolutionTableTest, TestBasic) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, - SpecialFields::RowId().field_, fields_[2]}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, + SpecialFields::RowId().field_, fields_[2]}), + R"([ ["a", 1, 2, 0, "c"] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, - expected_row_tracking_array)); + ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, + expected_row_tracking_array)); - // read score but not indexed split - ASSERT_NOK_WITH_MSG( - ScanAndRead(table_path, {"f0", "f1", "_INDEX_SCORE"}, expected_row_tracking_array, - /*predicate=*/nullptr, - /*row_ranges=*/{}), - "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit"); - } + // read score but not indexed split + ASSERT_NOK_WITH_MSG( + ScanAndRead(table_path, {"f0", "f1", "_INDEX_SCORE"}, expected_row_tracking_array, + /*predicate=*/nullptr, + /*row_ranges=*/{}), + "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit"); +} + +TEST_P(DataEvolutionTableTest, TestCommitConflictOnOverlappedRowIdAndWriteColumns) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Snapshot 1: initialize row id range [0, 0]. + std::vector init_write_cols = {"f0", "f1", "f2"}; + auto init_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ + [1, "a", "b"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto init_msgs, WriteArray(table_path, init_write_cols, init_array)); + ASSERT_OK(Commit(table_path, init_msgs)); + + // Snapshot 2: update f2 at row id 0. + std::vector write_cols = {"f2"}; + auto src_array_1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[2]}), R"([ + ["c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, WriteArray(table_path, write_cols, src_array_1)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs_1); + ASSERT_OK(Commit(table_path, commit_msgs_1)); + + // Snapshot 3 attempt: update f2 at row id 0 again, and check history from snapshot 1. + // This should conflict with snapshot 2 because row-id range and write columns overlap. + auto src_array_2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[2]}), R"([ + ["d"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, WriteArray(table_path, write_cols, src_array_2)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs_2); + ASSERT_NOK_WITH_MSG( + CommitWithRowIdCheckFromSnapshot(table_path, commit_msgs_2, + /*row_id_check_from_snapshot=*/1), + "multiple 'MERGE INTO' operations have encountered conflicts, updating the same file"); } TEST_P(DataEvolutionTableTest, TestMultipleAppends) { @@ -417,17 +502,16 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppends) { /*predicate=*/nullptr, /*row_ranges=*/row_ranges)); } - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, "a", "b", 0, 1], [1, "a", "b", 1, 1], [1, "a", "b", 2, 1], @@ -441,11 +525,10 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppends) { [1, "a", "b", 10, 2], [2, "c", "d", 11, 4] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) { @@ -492,24 +575,165 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, "a", "b", 0, 3] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); +} + +TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) { + if (FileFormat() == "avro") { + return; } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("map1", map_type), + arrow::field("map2", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, FileFormat()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"fields.map1.map.storage-layout", "shared-shredding"}, + {"fields.map1.map.shared-shredding.max-columns", "1"}, + {"fields.map2.map.storage-layout", "shared-shredding"}, + {"fields.map2.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + std::vector write_cols0 = {"id", "map1"}; + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), R"([ + [1, [["a", 10], ["b", 20]]], + [11, [["a", 11], ["b", 21]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, write_cols0, src_array0)); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + std::vector write_cols1 = {"id", "map2"}; + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), R"([ + [2, [["c", 30], ["d", 40]]], + [12, [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, write_cols1, src_array1)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + std::vector write_cols2 = {"map1"}; + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}), R"([ + [[["b", 200], ["a", 100]]], + [[["b", 201], ["a", 101]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, write_cols2, src_array2)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // Read all columns and merge values from all partial files. + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, [["a", 100], ["b", 200]], [["c", 30], ["d", 40]]], + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // Read a subset of columns and recall only the requested shared-shredding MAP column. + auto expected_column_pruned_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), R"([ + [2, [["c", 30], ["d", 40]]], + [12, [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"id", "map2"}, expected_column_pruned_array)); + + // Read selected keys from both shared-shredding MAP columns after partial overwrite merge. + { + auto map1_selected_keys = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"b"}); + auto map2_selected_keys = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"d"}); + auto read_schema = arrow::schema({ + fields[0], + fields[1]->WithMetadata(map1_selected_keys), + fields[2]->WithMetadata(map2_selected_keys), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected_type = arrow::struct_({ + SpecialFields::ValueKind().field_, + fields[0], + fields[1], + fields[2], + }); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 2, [["b", 200]], [["d", 40]]], + [0, 12, [["b", 201]], [["d", 41]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(expected_chunked->Equals(actual)) + << "actual=" << actual->ToString() << "\nexpected=" << expected_chunked->ToString(); + } + + // Read a subset of rows after merging values from all partial files. + auto expected_partial_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_partial_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1l, 1l)})); + + // Read row tracking fields and verify the latest partial overwrite sequence number. + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ + [2, [["a", 100], ["b", 200]], [["c", 30], ["d", 40]], 0, 3], + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]], 1, 3] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"id", "map1", "map2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestNullValues) { @@ -561,24 +785,22 @@ TEST_P(DataEvolutionTableTest, TestNullValues) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, null, "c", 0, 2] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestMultipleAppendsDifferentFirstRowIds) { @@ -644,25 +866,23 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppendsDifferentFirstRowIds) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ - fields_[0], - fields_[1], - fields_[2], - SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_, - }), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({ + fields_[0], + fields_[1], + fields_[2], + SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_, + }), + R"([ [1, "a", "b", 0, 1], [2, "c", "d", 1, 3] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestMoreData) { @@ -714,7 +934,7 @@ TEST_P(DataEvolutionTableTest, TestMoreData) { TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { std::map options = { {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "false"}, @@ -735,21 +955,19 @@ TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols0, src_array0)); ASSERT_OK(Commit(table_path, commit_msgs)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, - SpecialFields::RowId().field_, fields_[2]}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_, + SpecialFields::RowId().field_, fields_[2]}), + R"([ ["a", 1, 1, 0, "b"], ["c", 2, 1, 1, "d"] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestExternalPath) { @@ -760,7 +978,7 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) { std::map options = { {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -803,21 +1021,19 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ ["a", 10, "b", 0, 2], ["c", 20, "d", 1, 2] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) { @@ -881,10 +1097,9 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) { .ValueOrDie()); ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); - { - // test only read partition fields - auto expected_array_only_partition = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ + // test only read partition fields + auto expected_array_only_partition = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ ["2024"], ["2024"], ["2024"], @@ -892,15 +1107,15 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) { ["2025"], ["2025"] ])") - .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1"}, expected_array_only_partition)); + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f1"}, expected_array_only_partition)); - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ [1, "2024", "c1", 0, 2], [2, "2024", "c2", 1, 2], [3, "2024", "c3", 2, 2], @@ -908,17 +1123,17 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) { [null, "2025", "d2", 4, 3], [null, "2025", "d3", 5, 3] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); - // read only read partition fields and row tracking - auto expected_partition_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[1], SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_}), - R"([ + // read only read partition fields and row tracking + auto expected_partition_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[1], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ ["2024", 0, 2], ["2024", 1, 2], ["2024", 2, 2], @@ -926,11 +1141,10 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) { ["2025", 4, 3], ["2025", 5, 3] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f1", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_partition_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f1", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_partition_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestWithPartitionWithoutPartitionFieldsInFile) { @@ -992,32 +1206,30 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionWithoutPartitionFieldsInFile) { /*row_ranges=*/row_ranges)); } - { - // read with row tracking - auto expected_row_tracking_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_, - SpecialFields::SequenceNumber().field_}), - R"([ + // read with row tracking + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ [1, "2024", "c1", 0, 2], [2, "2024", "c2", 1, 2], [3, "2024", "c3", 2, 2] ])") - .ValueOrDie()); + .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, - expected_row_tracking_array)); - } + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); } TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } std::vector partition_keys = {"f1"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; @@ -1184,7 +1396,7 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { } TEST_P(DataEvolutionTableTest, TestAlterTable) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1281,7 +1493,7 @@ TEST_P(DataEvolutionTableTest, TestAlterTable) { } TEST_P(DataEvolutionTableTest, TestReadCompactFiles) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1311,7 +1523,7 @@ TEST_P(DataEvolutionTableTest, TestReadCompactFiles) { } TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1392,7 +1604,7 @@ TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) { } TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1533,7 +1745,7 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) { } TEST_P(DataEvolutionTableTest, TestPredicate) { - if (GetParam() == "avro") { + if (FileFormat() == "avro") { // Avro does not have stats. return; } @@ -1674,7 +1886,7 @@ TEST_P(DataEvolutionTableTest, TestIOException) { TEST_P(DataEvolutionTableTest, TestWithRowIds) { std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1836,7 +2048,7 @@ TEST_P(DataEvolutionTableTest, TestWithRowIds) { /*predicate=*/nullptr, /*row_ranges=*/row_ranges)); } - if (GetParam() == "avro") { + if (FileFormat() == "avro") { // Avro does not support stats. return; } @@ -1896,15 +2108,17 @@ TEST_P(DataEvolutionTableTest, TestWithRowIds) { } } -std::vector GetTestValuesForDataEvolutionTableTest() { - std::vector values; - values.emplace_back("parquet"); +std::vector GetTestValuesForDataEvolutionTableTest() { + std::vector values; + for (bool enable_snapshot_live_manifest_cache : {false, true}) { + values.emplace_back("parquet", enable_snapshot_live_manifest_cache); #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc"); + values.emplace_back("orc", enable_snapshot_live_manifest_cache); #endif #ifdef PAIMON_ENABLE_AVRO - values.emplace_back("avro"); + values.emplace_back("avro", enable_snapshot_live_manifest_cache); #endif + } return values; } diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 3091dc48..ccb9ce32 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -47,6 +47,12 @@ namespace paimon::test { using ParamType = std::tuple; /// This is a sdk end-to-end test for global index. class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInterface { + static const std::map& LegacyBitmapTestOptions() { + static const std::map options = { + {"bitmap-global-index.legacy-format.enabled-for-testing", "true"}}; + return options; + } + void SetUp() override { file_format_ = std::get<0>(GetParam()); dir_ = UniqueTestDirectory::Create("local"); @@ -63,13 +69,17 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter void CreateTable(const std::vector& partition_keys, const std::shared_ptr& schema, const std::map& options) const { + std::map test_options = options; + for (const auto& [key, value] : LegacyBitmapTestOptions()) { + test_options[key] = value; + } ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir_->Str(), {}, fs_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, partition_keys, - /*primary_keys=*/{}, options, + /*primary_keys=*/{}, test_options, /*ignore_if_exists=*/false)); } @@ -197,7 +207,9 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate).WithFileSystem(fs_); + read_context_builder.SetReadFieldNames(read_schema) + .SetPredicate(predicate) + .WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); @@ -325,8 +337,76 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) { ASSERT_NOK_WITH_MSG( WriteIndex(table_path, /*partition_filters=*/{}, "f1", "lumina", /*options=*/lumina_options, Range(0, 1)), - "invalid input array in LuminaIndexWriter, length of field array [1] multiplied " - "dimension [3] must match length of field value array [4]"); + "invalid input array in LuminaIndexWriter, vector at row [0] has length [4], " + "expected dimension [3]"); +} + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags) { + if (file_format_ != "orc") { + GTEST_SKIP() << "ORC-only dictionary encoding case"; + } + + arrow::FieldVector fields = {arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8()))}; + std::shared_ptr schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"}])"}}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {"orc.read.enable-lazy-decoding", "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::shared_ptr src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[[0.0, 0.0, 0.0, 0.0], "red", ["hot", "common"]], +[[0.0, 1.0, 0.0, 1.0], "blue", ["cold", "common"]], +[[1.0, 0.0, 1.0, 0.0], "red", ["cold"]], +[[1.0, 1.0, 1.0, 1.0], "blue", ["hot"]] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> commit_msgs, + WriteArray(table_path, schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "embedding", "lumina", + lumina_options, Range(0, 3))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::vector> lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/4, query, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0,2}, scores: {4.21,2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "hot", 3)}), + "row ids: {0,3}, scores: {4.21,0.01}"); } TEST_P(GlobalIndexTest, TestWriteIndex) { @@ -469,10 +549,11 @@ TEST_P(GlobalIndexTest, TestScanIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader // test f0 field ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); @@ -639,10 +720,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithSpecificSnapshot) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; // snapshot 2 has f0 index - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/2l, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/2l, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader // test f0 field ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); @@ -688,10 +770,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithSpecificSnapshotWithNoIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; // snapshot 1 has no index - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/1l, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/1l, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); ASSERT_EQ(index_readers.size(), 0u); @@ -712,10 +795,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithRange) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); auto global_index_scan_impl = std::dynamic_pointer_cast(global_index_scan); { // test index reader @@ -752,10 +836,10 @@ TEST_P(GlobalIndexTest, TestScanIndexWithPartition) { "/append_with_global_index_with_partition.db/append_with_global_index_with_partition"; auto check_result = [&](const std::optional>>& partitions) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, partitions, - /*options=*/{}, fs_, /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + partitions, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 4)})); @@ -808,10 +892,11 @@ TEST_P(GlobalIndexTest, TestScanUnregisteredIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); ASSERT_EQ(index_readers.size(), 0u); @@ -1133,6 +1218,158 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { ASSERT_FALSE(typed_result->bitmap_.Contains(7)); } } + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { + arrow::FieldVector fields = {arrow::field("name", arrow::utf8()), + arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}; + auto schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"range","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::vector write_cols = schema->field_names(); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["row0", [0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], +["row1", null, "red", ["vip"], 8.0, [0.8], 7, [9]], +["row2", [0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], +["row3", [1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], +["row4", [1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + ScanData(table_path, /*partition_filters=*/{})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index_commit_msg, + GlobalIndexWriteTask::WriteIndex( + table_path, "embedding", "lumina", + std::make_shared(split, std::vector({Range(0, 4)})), + /*options=*/lumina_options, pool_, fs_)); + + std::shared_ptr index_commit_msg_impl = + std::dynamic_pointer_cast(index_commit_msg); + ASSERT_TRUE(index_commit_msg_impl); + const auto& new_index_files = index_commit_msg_impl->GetNewFilesIncrement().NewIndexFiles(); + ASSERT_EQ(new_index_files.size(), 1u); + ASSERT_EQ(new_index_files[0]->IndexType(), "lumina"); + ASSERT_EQ(new_index_files[0]->RowCount(), 5); + const std::optional& global_index_meta = + new_index_files[0]->GetGlobalIndexMeta(); + ASSERT_TRUE(global_index_meta); + std::string expected_index_meta_json = + R"({"distance.metric":"l2","encoding.type":"rawf32","extension.build.tag.tag_schema":"[{\"key_name\":\"color\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"labels\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"price\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"scores\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"category\",\"type\":\"enum\",\"value_type\":\"int32\"},{\"key_name\":\"category_ids\",\"type\":\"enum\",\"value_type\":\"int32\"}]","index.dimension":"4","index.type":"bruteforce","search.parallel_number":"10"})"; + GlobalIndexMeta expected_global_index_meta( + /*row_range_start=*/0, /*row_range_end=*/4, /*index_field_id=*/1, + /*extra_field_ids=*/std::optional>({2, 3, 4, 5, 6, 7}), + std::make_shared(expected_index_meta_json, pool_.get())); + ASSERT_EQ(global_index_meta.value(), expected_global_index_meta); + + ASSERT_OK(Commit(table_path, {index_commit_msg})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check_with_filter = [&](VectorSearch::PreFilter pre_filter, + const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/5, query, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected); + }; + + search_and_check(/*predicate=*/nullptr, "row ids: {0,2,3,4}, scores: {4.21,2.01,2.21,0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + "row ids: {3}, scores: {2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/3, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::GreaterOrEqual(/*field_index=*/5, /*field_name=*/"scores", + FieldType::FLOAT, Literal(0.5f)), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"category", + FieldType::INT, Literal(7)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::In(/*field_index=*/7, /*field_name=*/"category_ids", + FieldType::INT, {Literal(9)}), + "row ids: {4}, scores: {0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + "row ids: {}, scores: {}"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, + "row ids: {0}, scores: {4.21}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, + "row ids: {}, scores: {}"); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/4, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, "row ids: {0,4}, scores: {4.21,0.01}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, + "row ids: {4}, scores: {0.01}"); + } +} #endif TEST_P(GlobalIndexTest, TestDataEvolutionBatchScan) { @@ -2133,6 +2370,22 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) { /*pre_filter=*/std::nullopt))); ASSERT_TRUE(index_result->ToString().find("row ids: {3}") != std::string::npos); } + { + ASSERT_OK_AND_ASSIGN(auto index_result, + index_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos); + } + { + ASSERT_OK_AND_ASSIGN(auto index_result, + index_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos); + } } TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) { @@ -2410,6 +2663,100 @@ TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndex) { } } +TEST_P(GlobalIndexTest, TestBTreeEmptyStringKeyPredicates) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + std::vector write_cols = schema->field_names(); + + // Null keys are tracked separately. The non-null keys remain monotonically increasing. + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +[null, 10, 0, 10.0], +["", 20, 1, 20.0], +["abc", 30, 2, 30.0] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "btree", + /*options=*/{}, Range(0, 2))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, /*options=*/{}, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto index_readers, + global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt)); + ASSERT_EQ(index_readers.size(), 1u); + auto index_reader = index_readers[0]; + + Literal empty(FieldType::STRING, "", 0); + Literal abc(FieldType::STRING, "abc", 3); + Literal missing(FieldType::STRING, "missing", 7); + + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(missing)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitNotEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIsNull()); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{0}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIsNotNull()); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1,2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIn({empty, abc, missing})); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1,2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitNotIn({empty, missing})); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitLessThan(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitLessOrEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitGreaterThan(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitGreaterOrEqual(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } +} + TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { // BTree index with partitioned table. Each partition's data is sorted by f0 independently. auto schema = arrow::schema(fields_); @@ -2597,7 +2944,8 @@ TEST_P(GlobalIndexTest, TestBTreeWithPartitionAndCustomExecutor) { /*options=*/{}, Range(5, 7))); // Create a GlobalIndexScan with an explicit 8-thread executor - std::shared_ptr executor = CreateDefaultExecutor(/*thread_count=*/8); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(/*thread_count=*/8)); ASSERT_OK_AND_ASSIGN( std::shared_ptr global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp new file mode 100644 index 00000000..b77d6bda --- /dev/null +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -0,0 +1,1486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/commit_context.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/fs/file_system_factory.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/dict_array_converter.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" + +namespace paimon { +class DataSplit; +class RecordBatch; +} // namespace paimon + +namespace paimon::test { + +class NestedColumnPruningInteTest : public ::testing::Test, + public ::testing::WithParamInterface { + void SetUp() override { + file_format_ = GetParam(); + dir_ = UniqueTestDirectory::Create("local"); + test_dir_ = dir_->Str(); + table_path_ = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + } + void TearDown() override { + dir_.reset(); + } + + void AssertChunkedArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) const { + arrow::EqualOptions equal_options = arrow::EqualOptions::Defaults(); + bool is_equal = expected->Equals(actual, equal_options.diff_sink(&std::cout)); + if (!is_equal) { + std::cout << "[expected_type] " << expected->type()->ToString() << std::endl; + std::cout << "[actual_type] " << actual->type()->ToString() << std::endl; + std::cout << "[expected] " << expected->ToString() << std::endl; + std::cout << "[actual] " << actual->ToString() << std::endl; + } + ASSERT_TRUE(is_equal); + } + + void ScanReadAndCheck(const std::string& table_path, + const std::shared_ptr& expected_schema, + const std::string& expected_json, + const std::shared_ptr& predicate = nullptr) const { + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*expected_schema, c_schema.get()).ok()); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetReadSchema(std::move(c_schema)); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = expected_schema->fields(); + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_type = arrow::struct_(expected_fields); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json).ValueOrDie()); + AssertChunkedArrayEquals(expected, actual); + } + + protected: + std::string file_format_; + std::string test_dir_; + std::string table_path_; + std::unique_ptr dir_; +}; + +// Test: Table has struct field with 3 sub-fields, read only 1 sub-field via SetReadSchema. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) { + // Table schema: f0 (int32), f1 (struct{a: int32, b: utf8, c: float64}) + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + arrow::field("c", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data + std::string data = R"([ + [1, [10, "hello", 1.1]], + [2, [20, "world", 2.2]], + [3, [30, "foo", 3.3]], + [4, [40, "bar", 4.4]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Build projected schema: only read f0 (full) and f1.a (sub-field of struct) + auto pruned_struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [10]], + [0, 2, [20]], + [0, 3, [30]], + [0, 4, [40]] + ])"); +} + +// Test: Projecting a STRUCT column as empty struct should return this column +// as all null values. +TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullColumn) { + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]], + [3, [33, "z"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Project f1 as empty struct. + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({})), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, null], + [0, 2, null], + [0, 3, null] + ])"); +} + +// Test: Two top-level struct columns have the same nested field name; projection should +// distinguish by parent column. +TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructColumns) { + // Table schema: f0 (int32), s0 (struct{f1: int32, a: utf8}), s1 (struct{f1: int32, b: utf8}) + auto s0_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("a", arrow::utf8()), + }); + auto s1_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("s0", s0_type), + arrow::field("s1", s1_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "left-1"], [101, "right-1"]], + [2, [22, "left-2"], [202, "right-2"]], + [3, [33, "left-3"], [303, "right-3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Project only s0.f1 and s1.f1; both nested field names are identical. + auto projected_s0 = arrow::struct_({arrow::field("f1", arrow::int32())}); + auto projected_s1 = arrow::struct_({arrow::field("f1", arrow::int32())}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("s0", projected_s0), + arrow::field("s1", projected_s1), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [11], [101]], + [0, 2, [22], [202]], + [0, 3, [33], [303]] + ])"); +} + +// Test: Querying only non-existent struct sub-fields should fail fast. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { + // Table schema: f0 (int32), f1 (struct{f1: int32, f2: utf8, f3: float64}) + auto struct_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "a", 1.1]], + [2, [22, "b", 2.2]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Query struct sub-fields that do not exist in table schema. + auto projected_struct_type = arrow::struct_({ + arrow::field("f4", arrow::int32()), + arrow::field("f5", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "does not support schema evolution inside struct"); +} + +// Test: Querying a mix of existent and non-existent struct sub-fields should fail fast. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { + // Table schema: f0 (int32), f1 (struct{f1: int32, f2: utf8, f3: float64}) + auto struct_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "a", 1.1]], + [2, [22, "b", 2.2]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Query struct sub-fields f2,f3,f4 where f4 does not exist in table schema. + auto projected_struct_type = arrow::struct_({ + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + arrow::field("f4", arrow::int32()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "does not support schema evolution inside struct"); +} + +// Test: Nested schema divergence (simulated evolution mismatch) must fail fast +// instead of silently skipping nested fields. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFail) { + // File schema (old): f1.a is INT32. + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Persist schema-2 on disk so table latest schema diverges from old data files. + std::string file_system_identifier = "local"; + auto fs_iter = options.find(Options::FILE_SYSTEM); + if (fs_iter != options.end()) { + file_system_identifier = StringUtils::ToLowerCase(fs_iter->second); + } + ASSERT_OK_AND_ASSIGN(auto file_system, + FileSystemFactory::Get(file_system_identifier, table_path_, options)); + std::shared_ptr schema_fs(std::move(file_system)); + + SchemaManager schema_manager(schema_fs, table_path_); + ASSERT_OK_AND_ASSIGN(auto latest_schema_opt, schema_manager.Latest()); + ASSERT_TRUE(latest_schema_opt.has_value()); + auto latest_schema = latest_schema_opt.value(); + + auto schema_v2_arrow = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + ASSERT_OK_AND_ASSIGN( + auto schema_v2, + TableSchema::Create(/*schema_id=*/latest_schema->Id() + 1, schema_v2_arrow, + latest_schema->PartitionKeys(), latest_schema->PrimaryKeys(), + latest_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto schema_v2_json, schema_v2->ToJsonString()); + auto schema_v2_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(schema_v2->Id())); + ASSERT_OK(schema_fs->AtomicStore(schema_v2_path, schema_v2_json)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), + "PruneDataType nested field type mismatch for 'a': read string vs " + "data int32"); +} + +// Test: With SetReadSchema using the new schema, context build should pass, +// and mismatch against old file type should be rejected in reader creation. +TEST_P(NestedColumnPruningInteTest, + QueryStructSubFieldsWithTypeMismatchAndSetReadSchemaFailAtContext) { + // File schema (old): f1.a is INT32. + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Persist schema-2 on disk so table latest schema diverges from old data files. + std::string file_system_identifier = "local"; + auto fs_iter = options.find(Options::FILE_SYSTEM); + if (fs_iter != options.end()) { + file_system_identifier = StringUtils::ToLowerCase(fs_iter->second); + } + ASSERT_OK_AND_ASSIGN(auto file_system, + FileSystemFactory::Get(file_system_identifier, table_path_, options)); + std::shared_ptr schema_fs(std::move(file_system)); + + SchemaManager schema_manager(schema_fs, table_path_); + ASSERT_OK_AND_ASSIGN(auto latest_schema_opt, schema_manager.Latest()); + ASSERT_TRUE(latest_schema_opt.has_value()); + auto latest_schema = latest_schema_opt.value(); + + auto schema_v2_arrow = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + ASSERT_OK_AND_ASSIGN( + auto schema_v2, + TableSchema::Create(/*schema_id=*/latest_schema->Id() + 1, schema_v2_arrow, + latest_schema->PartitionKeys(), latest_schema->PrimaryKeys(), + latest_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto schema_v2_json, schema_v2->ToJsonString()); + auto schema_v2_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(schema_v2->Id())); + ASSERT_OK(schema_fs->AtomicStore(schema_v2_path, schema_v2_json)); + + // User-provided read schema uses latest nested type (a:string). + auto projected_schema = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), + "PruneDataType nested field type mismatch for 'a': read string vs " + "data int32"); +} + +// Test: Read only top-level fields, skip struct entirely. +TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) { + auto struct_type = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + arrow::field("f2", arrow::float64()), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [100, [1, "aa"], 0.1], + [200, [2, "bb"], 0.2], + [300, [3, "cc"], 0.3] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Only read f0 and f2, skip f1 entirely. + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f2", arrow::float64()), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 100, 0.1], + [0, 200, 0.2], + [0, 300, 0.3] + ])"); +} + +// Test: Nested struct — prune sub-fields of a struct inside another struct. +TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner: struct{x: int64, y: utf8}}) + auto inner_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner", inner_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, "aaa"]]], + [2, [20, [200, "bbb"]]], + [3, [30, [300, "ccc"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Projected: f0, f1{inner{x}} — skip f1.a and f1.inner.y + auto pruned_inner = arrow::struct_({ + arrow::field("x", arrow::int64()), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner", pruned_inner), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [[100]]], + [0, 2, [[200]]], + [0, 3, [[300]]] + ])"); +} + +// Test: Nested projected schema with special fields under row tracking. +TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner: struct{x: int64, y: utf8}}) + auto inner_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner", inner_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, "aaa"]]], + [2, [20, [200, "bbb"]]], + [3, [30, [300, "ccc"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + + // Projected: f0, f1{inner{x}}, _SEQUENCE_NUMBER, _ROW_ID + auto pruned_inner = arrow::struct_({ + arrow::field("x", arrow::int64()), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner", pruned_inner), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + arrow::field("_ROW_ID", arrow::int64()), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + ASSERT_EQ(read_result->num_chunks(), 1); + auto result_array = std::dynamic_pointer_cast(read_result->chunk(0)); + ASSERT_TRUE(result_array); + + ASSERT_TRUE(result_array->GetFieldByName("_SEQUENCE_NUMBER")); + ASSERT_TRUE(result_array->GetFieldByName("_ROW_ID")); + auto nested_col = result_array->GetFieldByName("f1"); + ASSERT_TRUE(nested_col); + + auto expected_nested_type = arrow::struct_({ + arrow::field("inner", arrow::struct_({arrow::field("x", arrow::int64())})), + }); + ASSERT_TRUE(nested_col->type()->Equals(expected_nested_type)); + + auto expected_nested_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_nested_type, R"([ + [[100]], + [[200]], + [[300]] + ])") + .ValueOrDie(); + ASSERT_TRUE(nested_col->Equals(expected_nested_array)); +} + +// Test: Table has MAP field, read with selected keys filter. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) { + // Table schema: f0 (int32), f1 (map) + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data: each row has a map with keys "a", "b", "c" + std::string data = R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 100], ["c", 300]]], + [3, [["b", 200], ["c", 400], ["d", 500]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Build projected schema: read f0 and f1 with selected keys "a,c" + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + // Expected: only keys "a" and "c" remain in each map + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [["a", 10], ["c", 30]]], + [0, 2, [["a", 100], ["c", 300]]], + [0, 3, [["c", 400]]] + ])"); +} + +// Test: Selected-keys metadata on MAP nested inside STRUCT should be applied. +TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("tag", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], "r1"]], + [2, [[ ["a", 100], ["c", 300] ], "r2"]], + [3, [[ ["b", 200], ["c", 400], ["d", 500] ], "r3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [[ ["a", 10], ["c", 30] ]]], + [0, 2, [[ ["a", 100], ["c", 300] ]]], + [0, 3, [[ ["c", 400] ]]] + ])"); +} + +// Test: Partial STRUCT sub-field recall where one recalled child is MAP with selected keys. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKeys) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("keep", arrow::int64()), + arrow::field("drop", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], 1001, "x1"]], + [2, [[ ["a", 100], ["c", 300] ], 1002, "x2"]], + [3, [[ ["b", 200], ["c", 400], ["d", 500] ], 1003, "x3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("keep", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], + [0, 2, [[ ["a", 100], ["c", 300] ], 1002]], + [0, 3, [[ ["c", 400] ], 1003]] + ])"); +} + +// Test: Null semantics should be preserved when pruning STRUCT sub-fields and +// applying selected-keys filtering on nested MAP. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKeysAndNulls) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("keep", arrow::int64()), + arrow::field("drop", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], 1001, "x1"]], + [2, null], + [3, [null, 1003, "x3"]], + [4, [[ ["b", 200], ["c", 400], ["d", 500] ], null, "x4"]], + [5, [[ ["a", 500], ["c", null] ], 1005, "x5"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("keep", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], + [0, 2, null], + [0, 3, [null, 1003]], + [0, 4, [[ ["c", 400] ], null]], + [0, 5, [[ ["a", 500], ["c", null] ], 1005]] + ])"); +} + +// Test: MAP_SELECTED_KEYS metadata value is empty string, select empty-string map key. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) { + // Table schema: f0 (int32), f1 (map) + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data: each row has a map that may contain empty-string key. + std::string data = R"([ + [1, [["", 9], ["a", 10], ["c", 30]]], + [2, [["a", 100], ["", 99], ["c", 300]]], + [3, [["b", 200], ["c", 400], ["d", 500]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Build projected schema: read f0 and f1 with selected keys metadata set to empty string. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {""}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + // Expected: only empty-string key remains. + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [["", 9]]], + [0, 2, [["", 99]]], + [0, 3, []] + ])"); +} + +// Test: MAP_SELECTED_KEYS output map entry order should follow selected key order. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data with map key order different from selected key order. + std::string data = R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 100], ["c", 300]]], + [3, [["c", 400], ["a", 500], ["d", 600]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Query key order is c,a and output should follow this order. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a"}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [["c", 30], ["a", 10]]], + [0, 2, [["c", 300], ["a", 100]]], + [0, 3, [["c", 400], ["a", 500]]] + ])"); +} + +TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { + if (file_format_ == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto info_type = arrow::struct_({ + arrow::field("score", arrow::int64()), + arrow::field("label", arrow::utf8()), + arrow::field("drop", arrow::utf8()), + }); + auto payload_type = arrow::struct_({ + arrow::field("attrs", map_type), + arrow::field("info", info_type), + arrow::field("note", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("id", arrow::int32()), + arrow::field("payload", payload_type), + arrow::field("category", arrow::utf8()), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + auto write_one_row = [&](const std::string& data) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + return file_store_write->Write(std::move(batch)); + }; + + ASSERT_OK(write_one_row( + R"([[1, [[["a", 10], ["b", 20], ["c", 30]], [1001, "low", "x"], "n1"], "hot"]])")); + ASSERT_OK(write_one_row( + R"([[12, [[["a", 100], ["c", 300], ["d", 400]], [1002, "mid", "y"], "n2"], "warm"]])")); + ASSERT_OK(write_one_row( + R"([[21, [[["b", 200], ["c", 500], ["a", 600]], [1003, "high", "z"], "n3"], "cold"]])")); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + file_store_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/0)); + ASSERT_OK(file_store_write->Close()); + + CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); + + // Read selected MAP keys together with nested STRUCT sub-fields. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a"}); + auto selected_payload_type = arrow::struct_({ + arrow::field("attrs", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("info", arrow::struct_({arrow::field("score", arrow::int64())})), + }); + auto selected_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("payload", selected_payload_type), + }); + + ScanReadAndCheck(table_path_, selected_schema, R"([ + [0, 1, [[["c", 30], ["a", 10]], [1001]]], + [0, 12, [[["c", 300], ["a", 100]], [1002]]], + [0, 21, [[["c", 500], ["a", 600]], [1003]]] + ])"); + + // Read only part of top-level columns and part of nested STRUCT fields. + auto partial_payload_type = arrow::struct_({ + arrow::field("info", arrow::struct_({arrow::field("label", arrow::utf8())})), + }); + auto partial_schema = arrow::schema({ + arrow::field("payload", partial_payload_type), + arrow::field("category", arrow::utf8()), + }); + + ScanReadAndCheck(table_path_, partial_schema, R"([ + [0, [["low"]], "hot"], + [0, [["mid"]], "warm"], + [0, [["high"]], "cold"] + ])"); + + // Read selected nested fields with predicate pushdown. + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(10)); + + ScanReadAndCheck(table_path_, selected_schema, R"([ + [0, 12, [[["c", 300], ["a", 100]], [1002]]], + [0, 21, [[["c", 500], ["a", 600]], [1003]]] + ])", + predicate); +} + +// Test: ORC dictionary-encoded map key/value should work with MAP_SELECTED_KEYS. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) { + if (file_format_ != "orc") { + GTEST_SKIP() << "ORC-only dictionary encoding case"; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::utf8()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1.0"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Low-cardinality map keys/values increase dictionary-encoding probability for ORC. + std::string data = R"([ + [1, [["a", "v1"], ["b", "v2"], ["c", "v3"]]], + [2, [["a", "v1"], ["c", "v3"], ["d", "v4"]]], + [3, [["a", "v1"], ["b", "v2"], ["e", "v5"]]], + [4, [["a", "v1"], ["c", "v3"], ["e", "v5"]]], + [5, [["a", "v1"], ["b", "v2"], ["c", "v3"]]], + [6, [["a", "v1"], ["c", "v3"], ["d", "v4"]]], + [7, [["a", "v1"], ["b", "v2"], ["e", "v5"]]], + [8, [["a", "v1"], ["c", "v3"], ["e", "v5"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_schema = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + ASSERT_OK_AND_ASSIGN( + auto decoded_result, + DictArrayConverter::ConvertDictArray(read_result->chunk(0), arrow::default_memory_pool())); + auto actual_chunked = std::make_shared(decoded_result); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::utf8())), + }); + auto expected_array = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 1, [["a", "v1"], ["c", "v3"]]], + [0, 2, [["a", "v1"], ["c", "v3"]]], + [0, 3, [["a", "v1"]]], + [0, 4, [["a", "v1"], ["c", "v3"]]], + [0, 5, [["a", "v1"], ["c", "v3"]]], + [0, 6, [["a", "v1"], ["c", "v3"]]], + [0, 7, [["a", "v1"]]], + [0, 8, [["a", "v1"], ["c", "v3"]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, actual_chunked); +} + +// Test: Deeper nested struct — prune sub-fields of a struct inside a struct inside another +// struct. +TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner1: struct{x: int64, inner2: struct{p: + // utf8, q: float64}}}) + auto inner2_struct = arrow::struct_({ + arrow::field("p", arrow::utf8()), + arrow::field("q", arrow::float64()), + }); + auto inner1_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("inner2", inner2_struct), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner1", inner1_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, ["ppp", 1.1]]]], + [2, [20, [200, ["qqq", 2.2]]]], + [3, [30, [300, ["rrr", 3.3]]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Projected: f0, f1{inner1{inner2{p}}} + auto pruned_inner2 = arrow::struct_({ + arrow::field("p", arrow::utf8()), + }); + auto pruned_inner1 = arrow::struct_({ + arrow::field("inner2", pruned_inner2), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner1", pruned_inner1), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + }; + auto projected_schema = arrow::schema(projected_fields); + + ScanReadAndCheck(table_path_, projected_schema, R"([ + [0, 1, [[[ "ppp" ]]]], + [0, 2, [[[ "qqq" ]]]], + [0, 3, [[[ "rrr" ]]]] + ])"); +} + +// Test: Nested pruning for LIST> in integration path. +TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { + auto list_elem_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + arrow::field("z", arrow::float64()), + }); + auto list_struct_type = arrow::list(arrow::field("item", list_elem_struct)); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", list_struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[100, "a", 1.1], [200, "b", 2.2]]], + [2, [[300, "c", 3.3]]], + [3, []] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto pruned_list_elem_struct = arrow::struct_({arrow::field("x", arrow::int64())}); + auto pruned_list_type = arrow::list(arrow::field("item", pruned_list_elem_struct)); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_list_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto create_reader_result = table_read->CreateReader(data_splits); + ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); +} + +std::vector GetTestValuesForNestedColumnPruningInteTest() { + std::vector values; + values.emplace_back("parquet"); +#ifdef PAIMON_ENABLE_ORC + values.emplace_back("orc"); +#endif + return values; +} + +INSTANTIATE_TEST_SUITE_P(FileFormats, NestedColumnPruningInteTest, + ::testing::ValuesIn(GetTestValuesForNestedColumnPruningInteTest())); + +} // namespace paimon::test diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index b9962209..47e0e203 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -44,11 +44,15 @@ #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/data_generator.h" #include "paimon/testing/utils/io_exception_helper.h" @@ -116,8 +120,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -136,8 +140,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -150,8 +154,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); } Result>> WriteArray( @@ -181,9 +185,6 @@ class PkCompactionInteTest : public ::testing::Test, Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { CommitContextBuilder commit_builder(table_path, "commit_user_1"); - std::map commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - commit_builder.SetOptions(commit_options); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, @@ -226,13 +227,23 @@ class PkCompactionInteTest : public ::testing::Test, void ScanAndVerify(const std::string& table_path, const arrow::FieldVector& fields, const std::map, std::string>& - expected_data_per_partition_bucket) { + expected_data_per_partition_bucket, + const std::shared_ptr& predicate = nullptr) { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); - ASSERT_OK_AND_ASSIGN( - std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( + Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); + std::vector> data_splits = result_plan->Splits(); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -255,8 +266,31 @@ class PkCompactionInteTest : public ::testing::Test, auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, splits, iter->second)); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(splits)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(data_type, iter->second).ValueOrDie(); + auto expected_chunk_array = std::make_shared(expected_array); + + bool success = expected_chunk_array->Equals(read_result); + if (!success) { + std::cout << "[expected_data_type]" << expected_chunk_array->type()->ToString() + << std::endl; + std::cout << "[actual_data_type]" << read_result->type()->ToString() << std::endl; + std::cout << "[expected]:" << expected_chunk_array->ToString() << std::endl; + std::cout << "[actual]: " << read_result->ToString() << std::endl; + } ASSERT_TRUE(success); } } @@ -320,6 +354,190 @@ class PkCompactionInteTest : public ::testing::Test, arrow::FieldVector fields_; }; +// Verify shared-shredding MAP can be read correctly after PK full compaction. +TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::vector primary_keys = {"id"}; + std::vector partition_keys = {}; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir_->Str(), arrow::schema(fields), partition_keys, + primary_keys, options, /*is_streaming_mode=*/true)); + + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 100], ["d", 400]]], + [3, [["e", 50]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", 100], ["d", 400]]], + [0, 2, [["c", 30]]], + [0, 3, [["e", 50]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("padding", arrow::utf8()), + }; + std::vector primary_keys = {"id"}; + std::vector partition_keys = {}; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_COMPRESSION, "none"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + }; + CreateTable(fields, partition_keys, primary_keys, options); + std::string table_path = TablePath(); + auto data_type = arrow::struct_(fields); + int64_t commit_id = 0; + std::string padding(2048, 'X'); + + { + // clang-format off + std::string json_data = R"([ +[1, [["a", 10], ["b", 20]], ")" + padding + R"("], +[2, [["c", 30]], ")" + padding + R"("], +[3, [["d", 40]], ")" + padding + R"("], +[4, null, ")" + padding + R"("], +[6, [["j", 60], ["k", 70]], ")" + padding + R"("], +[7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[8, [["o", 110]], ")" + padding + R"("] +])"; + // clang-format on + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json_data).ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + ASSERT_OK_AND_ASSIGN( + auto upgrade_msgs, + CompactAndCommit(table_path, {}, 0, /*full_compaction=*/true, commit_id++)); + ASSERT_FALSE(HasDeletionVectorIndexFiles(upgrade_msgs)) + << "Initial full compact should not produce DV index files"; + + { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + [1, [["a", 100], ["e", 500]], "u1"], + [5, [["h", 80]], "u5"] + ])") + .ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + [2, [["c", 300], ["f", 600], ["g", 700]], "u2"], + [5, [["h", 800], ["i", 900]], "u5-new"] + ])") + .ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + ASSERT_OK_AND_ASSIGN( + auto dv_compact_msgs, + CompactAndCommit(table_path, {}, 0, /*full_compaction=*/false, commit_id++)); + ASSERT_TRUE(HasDeletionVectorIndexFiles(dv_compact_msgs)) + << "Non-full compact should produce DV index files"; + + std::map, std::string> expected_data; + // clang-format off + expected_data[std::make_pair("", 0)] = R"([ +[0, 3, [["d", 40]], ")" + padding + R"("], +[0, 4, null, ")" + padding + R"("], +[0, 6, [["j", 60], ["k", 70]], ")" + padding + R"("], +[0, 7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[0, 8, [["o", 110]], ")" + padding + R"("], +[0, 1, [["a", 100], ["e", 500]], "u1"], +[0, 2, [["c", 300], ["f", 600], ["g", 700]], "u2"], +[0, 5, [["h", 800], ["i", 900]], "u5-new"] +])"; + // clang-format on + ScanAndVerify(table_path, fields, expected_data); + + // read with predicate and dv bitmap + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(6)); + std::map, std::string> expected_predicate_data; + // clang-format off + expected_predicate_data[std::make_pair("", 0)] = R"([ +[0, 7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[0, 8, [["o", 110]], ")" + padding + R"("] +])"; + // clang-format on + ScanAndVerify(table_path, fields, expected_predicate_data, predicate); +} + // Test: deduplicate merge engine with deletion vectors enabled. // Verifies that a non-full compact produces DV index files when level-0 files // overlap with high-level files, and that data is correct after DV compact and full compact. @@ -1997,13 +2215,21 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #1 must produce DV for Alice/Bob in L5"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Carol", 10, 0, 3.0, ")" + padding + R"("], + [0, "Dave", 10, 0, 4.0, ")" + padding + R"("], + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 20.0, "v2b"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 3: Write batch_3 (overlap Bob/Carol) → non-full compact. - // L0 merges to a lower intermediate level; DV marks Bob in the intermediate file - // from Step 2, and Carol in L5. + // DV marks Carol in L5. { auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Bob", 10, 0, 200.0, "v3b"], @@ -2015,8 +2241,16 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #2 must produce DV for Bob (intermediate) and Carol (L5)"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Dave", 10, 0, 4.0, ")" + padding + R"("], + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 200.0, "v3b"], + [0, "Carol", 10, 0, 300.0, "v3c"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 4: Write batch_4 (overlap Carol/Dave) → non-full compact. @@ -2032,8 +2266,16 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #3 must produce DV for Carol (intermediate) and Dave (L5)"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 200.0, "v3b"], + [0, "Carol", 10, 0, 3000.0, "v4c"], + [0, "Dave", 10, 0, 4000.0, "v4d"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 5: Write batch_5 (overlap Dave/Eve) → leave at L0 (no compact). @@ -2047,11 +2289,11 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK(WriteAndCommit(table_path, {{"f1", "10"}}, 0, array, commit_id++)); std::map, std::string> expected_data; expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], [0, "Alice", 10, 0, 10.0, "v2a"], [0, "Bob", 10, 0, 200.0, "v3b"], [0, "Carol", 10, 0, 3000.0, "v4c"], - [0, "Dave", 10, 0, 40000.0, "v5d"], - [0, "Eve", 10, 0, 50000.0, "v5e"] + [0, "Dave", 10, 0, 4000.0, "v4d"] ])"; ScanAndVerify(table_path, fields, expected_data); } @@ -2060,9 +2302,7 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN( auto final_compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/true, commit_id++)); - } - // Step 7: ScanAndVerify after full compact (globally sorted, all data in L5). - { + // Step 7: ScanAndVerify after full compact (globally sorted, all data in L5). std::map, std::string> expected_data; expected_data[std::make_pair("f1=10/", 0)] = R"([ [0, "Alice", 10, 0, 10.0, "v2a"], @@ -2559,8 +2799,8 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(9, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); diff --git a/test/inte/predicate_abi_inte_test.cpp b/test/inte/predicate_abi_inte_test.cpp new file mode 100644 index 00000000..3590eb22 --- /dev/null +++ b/test/inte/predicate_abi_inte_test.cpp @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Regression test for the Predicate cross-DSO RTTI/ABI fix (the out-of-line +// `Predicate::~Predicate` key function in common/predicate/predicate.cpp). +// +// `Predicate` is an abstract base with only pure virtuals. Without an +// out-of-line virtual ("key function"), the compiler has no single home for its +// vtable and typeinfo, so it emits them *weakly* in every translation unit that +// uses them. Combined with `-fvisibility=hidden`, separately linked modules +// (e.g. libpaimon.so vs libpaimon_parquet_file_format.so, or plugins loaded via +// dlopen) can each end up with their own `Predicate` typeinfo. A cross-module +// `dynamic_cast`/`dynamic_pointer_cast` then compares mismatched typeinfo and +// fails. Giving `Predicate` an out-of-line destructor anchors a single, +// exported definition of its RTTI that all other modules import. +// +// This file guards the fix on two levels: +// 1. Behavior: `Predicate` objects are created inside libpaimon.so and cast to +// derived types here in the test module. This catches the failure on +// toolchains that compare std::type_info by pointer (e.g. libc++) and in +// downstream builds that link plugins aggressively. +// 2. Symbol layout (Linux/ELF only): assert that `Predicate`'s typeinfo has a +// single home in libpaimon.so which the format plugins import, rather than +// each plugin emitting its own duplicate copy. This is the deterministic +// signature of the key function on glibc/libstdc++ toolchains, which merge +// weak typeinfo by name and therefore would not fail the behavioral cast +// even without the fix. + +#include + +#include "gtest/gtest.h" +#include "paimon/defs.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +// A leaf predicate built in libpaimon.so must survive dynamic_pointer_cast to +// LeafPredicate across the module boundary. +TEST(PredicateAbiInteTest, LeafPredicateCastsAcrossModule) { + std::shared_ptr predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + Literal(static_cast(5))); + ASSERT_NE(predicate, nullptr); + + std::shared_ptr leaf = std::dynamic_pointer_cast(predicate); + ASSERT_NE(leaf, nullptr) << "dynamic_pointer_cast failed across the module " + "boundary; Predicate RTTI is likely duplicated (missing key " + "function)"; + EXPECT_EQ(leaf->FieldName(), "f0"); + EXPECT_EQ(leaf->FieldIndex(), 0); + + // A leaf predicate must not be mistaken for a compound predicate. + EXPECT_EQ(std::dynamic_pointer_cast(predicate), nullptr); +} + +// A compound predicate built in libpaimon.so must survive dynamic_pointer_cast +// to CompoundPredicate, and its children must remain castable to LeafPredicate. +TEST(PredicateAbiInteTest, CompoundPredicateCastsAcrossModule) { + std::shared_ptr left = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + Literal(static_cast(5))); + std::shared_ptr right = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({left, right})); + + std::shared_ptr compound = + std::dynamic_pointer_cast(predicate); + ASSERT_NE(compound, nullptr) + << "dynamic_pointer_cast failed across the module boundary"; + ASSERT_EQ(compound->Children().size(), 2u); + + EXPECT_EQ(std::dynamic_pointer_cast(predicate), nullptr); + for (const auto& child : compound->Children()) { + EXPECT_NE(std::dynamic_pointer_cast(child), nullptr); + } +} + +} // namespace paimon::test + +#if defined(__linux__) && defined(__ELF__) +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace paimon::test { +namespace { + +// Mangled name of `typeinfo for paimon::Predicate`. +constexpr const char* kPredicateTypeInfoSymbol = "_ZTIN6paimon9PredicateE"; + +// Result of looking up a symbol in an ELF dynamic symbol table. +enum class DynSym { + kError, // could not read the file + kAbsent, // symbol not present in .dynsym + kDefined, // symbol has a definition in this module (st_shndx != SHN_UNDEF) + kImported, // symbol is undefined here and resolved from another module +}; + +// On-disk path of the loaded shared object whose path contains `needle`. +std::string FindLoadedModule(const char* needle) { + struct Ctx { + const char* needle; + std::string path; + } ctx{needle, {}}; + dl_iterate_phdr( + [](struct dl_phdr_info* info, size_t, void* data) -> int { + auto* c = static_cast(data); + if (info->dlpi_name != nullptr && std::strstr(info->dlpi_name, c->needle) != nullptr) { + c->path = info->dlpi_name; + return 1; + } + return 0; + }, + &ctx); + return ctx.path; +} + +// Whether `sym_name` is defined in, imported by, or absent from the dynamic +// symbol table of the ELF file at `path`. +// +// We look at whether the symbol is *defined* (has a section index) rather than +// at its binding (GLOBAL vs WEAK): a class with a key function has WEAK typeinfo +// on GCC but GLOBAL typeinfo on Clang, so the binding is not portable. What both +// compilers guarantee is that the key function makes the typeinfo a single +// definition that other modules import. +DynSym LookupDynSym(const std::string& path, const char* sym_name) { + int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) { + return DynSym::kError; + } + struct stat st{}; + if (::fstat(fd, &st) != 0) { + ::close(fd); + return DynSym::kError; + } + void* map = ::mmap(nullptr, static_cast(st.st_size), PROT_READ, MAP_PRIVATE, fd, 0); + ::close(fd); + if (map == MAP_FAILED) { + return DynSym::kError; + } + + const auto* base = static_cast(map); + const auto* ehdr = reinterpret_cast(base); + DynSym result = DynSym::kAbsent; + if (std::memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0 && ehdr->e_ident[EI_CLASS] == ELFCLASS64) { + const auto* shdrs = reinterpret_cast(base + ehdr->e_shoff); + for (int i = 0; i < ehdr->e_shnum; ++i) { + if (shdrs[i].sh_type != SHT_DYNSYM) { + continue; + } + const auto* syms = reinterpret_cast(base + shdrs[i].sh_offset); + const char* strtab = + reinterpret_cast(base + shdrs[shdrs[i].sh_link].sh_offset); + size_t count = shdrs[i].sh_size / sizeof(Elf64_Sym); + for (size_t s = 0; s < count; ++s) { + if (std::strcmp(strtab + syms[s].st_name, sym_name) == 0) { + result = syms[s].st_shndx == SHN_UNDEF ? DynSym::kImported : DynSym::kDefined; + break; + } + } + break; + } + } + ::munmap(map, static_cast(st.st_size)); + return result; +} + +} // namespace + +// Deterministic guard (compiler-agnostic across GCC and Clang): the out-of-line +// `~Predicate()` key function must give `Predicate`'s typeinfo a single home in +// libpaimon.so, which the format plugins then import instead of each emitting +// their own copy. Without the key function the typeinfo is weak and every +// module that casts a Predicate defines its own duplicate -- exactly the +// condition that breaks cross-DSO dynamic_cast. Reverting the fix flips this +// test from PASS to FAIL. +TEST(PredicateAbiInteTest, PredicateTypeInfoHasSingleHome) { + std::string core = FindLoadedModule("libpaimon.so"); + ASSERT_FALSE(core.empty()) << "could not locate loaded libpaimon.so"; + + // The core library owns the one definition of the typeinfo. + EXPECT_EQ(LookupDynSym(core, kPredicateTypeInfoSymbol), DynSym::kDefined) + << "typeinfo for paimon::Predicate must be defined in " << core; + + // A format plugin that casts predicates must import that definition rather + // than defining its own duplicate. The parquet plugin performs + // dynamic_pointer_cast/ on Predicate, so it + // references the typeinfo; with the key function that reference resolves to + // libpaimon.so (imported), without it the plugin carries its own weak copy. + std::string plugin = FindLoadedModule("libpaimon_parquet_file_format.so"); + ASSERT_FALSE(plugin.empty()) << "could not locate loaded libpaimon_parquet_file_format.so"; + + DynSym in_plugin = LookupDynSym(plugin, kPredicateTypeInfoSymbol); + ASSERT_NE(in_plugin, DynSym::kError) << "could not read " << plugin; + ASSERT_NE(in_plugin, DynSym::kAbsent) + << "typeinfo for paimon::Predicate is not referenced by " << plugin + << "; the parquet plugin is expected to cast Predicate across the DSO boundary"; + EXPECT_EQ(in_plugin, DynSym::kImported) + << "typeinfo for paimon::Predicate is defined inside " << plugin + << " instead of being imported from libpaimon.so; the Predicate RTTI is duplicated across " + "DSOs (the ~Predicate() key function is missing), which breaks cross-DSO dynamic_cast"; +} + +} // namespace paimon::test +#endif // defined(__linux__) && defined(__ELF__) diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index d00a0d70..9ebf2d60 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -16,8 +16,11 @@ * limitations under the License. */ +#include +#include #include #include +#include #include #include #include @@ -35,23 +38,31 @@ #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" +#include "paimon/core/snapshot.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/tag/tag.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" @@ -72,6 +83,7 @@ #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon::test { @@ -190,7 +202,7 @@ class ReadInteTest : public testing::Test, public ::testing::WithParamInterface< auto file_system = std::make_shared(); EXPECT_OK_AND_ASSIGN(auto input_stream, file_system->Open(split_file_name)); std::vector split_bytes(input_stream->Length().value_or(0), 0); - EXPECT_OK_AND_ASSIGN([[maybe_unused]] int32_t read_len, + EXPECT_OK_AND_ASSIGN([[maybe_unused]] int64_t read_len, input_stream->Read(split_bytes.data(), split_bytes.size())); EXPECT_OK(input_stream->Close()); @@ -268,17 +280,33 @@ std::vector StructFieldNames(const std::shared_ptr ReadSystemTable(const std::string& system_table_path, - const std::map& options) { + const std::map& options, + bool streaming_mode = false, + const std::shared_ptr& predicate = nullptr, + const std::vector& read_field_names = {}, + bool read_next_plan = false) { ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + if (read_next_plan) { + PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan()); + } ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + if (!read_field_names.empty()) { + read_context_builder.SetReadFieldNames(read_field_names); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, @@ -290,6 +318,17 @@ Result ReadSystemTable(const std::string& system_table_pa return SystemTableReadResult(std::move(batch_reader), result); } +Status WriteAndFullCompact(std::unique_ptr&& batch, int64_t commit_identifier, + TestHelper* helper) { + PAIMON_RETURN_NOT_OK(helper->write_->Write(std::move(batch))); + PAIMON_RETURN_NOT_OK( + helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + return helper->commit_->Commit(commit_messages, commit_identifier); +} + void AssertStructArrayEqualsJson(const std::shared_ptr& actual, const std::string& expected_json) { ASSERT_TRUE(actual); @@ -299,6 +338,18 @@ void AssertStructArrayEqualsJson(const std::shared_ptr& actu << "expected: " << expected->ToString() << "\nactual: " << actual->ToString(); } +Result CountDataFiles(const std::vector>& splits) { + int64_t file_count = 0; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("expected data split"); + } + file_count += data_split->GetFileList().size(); + } + return file_count; +} + } // namespace std::vector PrepareTestParam() { @@ -505,7 +556,7 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { ReadContextBuilder context_builder(path); context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.SetReadSchema({"dt"}); + context_builder.SetReadFieldNames({"dt"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.EnablePrefetch(param.enable_prefetch) .AddOption(Options::FILE_FORMAT, param.file_format) @@ -561,7 +612,7 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - std::string system_table_path = catalog->GetTableLocation(Identifier("db1", "tbl1$options")); + std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); @@ -616,6 +667,862 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { ASSERT_EQ(CollectStringMap(result), expected) << result->ToString(); } +TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) { + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["b", 2]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto snapshots_result, + ReadSystemTable(table_path + "$snapshots", options)); + auto snapshots_array = SingleStructChunk(snapshots_result); + ASSERT_EQ(StructFieldNames(snapshots_array), + (std::vector{ + "snapshot_id", "schema_id", "commit_user", "commit_identifier", "commit_kind", + "commit_time", "base_manifest_list", "delta_manifest_list", + "changelog_manifest_list", "total_record_count", "delta_record_count", + "changelog_record_count", "watermark", "next_row_id"})); + ASSERT_EQ(snapshots_array->length(), 2); + auto snapshot_id_array = + std::dynamic_pointer_cast(snapshots_array->field(0)); + auto commit_kind_array = + std::dynamic_pointer_cast(snapshots_array->field(4)); + auto commit_time_array = + std::dynamic_pointer_cast(snapshots_array->field(5)); + ASSERT_TRUE(snapshot_id_array); + ASSERT_TRUE(commit_kind_array); + ASSERT_TRUE(commit_time_array); + ASSERT_EQ(snapshot_id_array->Value(0), 1); + ASSERT_EQ(snapshot_id_array->Value(1), 2); + ASSERT_EQ(commit_kind_array->GetString(0), "APPEND"); + ASSERT_EQ(commit_kind_array->GetString(1), "APPEND"); + + ASSERT_OK_AND_ASSIGN(auto schemas_result, ReadSystemTable(table_path + "$schemas", options)); + auto schemas_array = SingleStructChunk(schemas_result); + ASSERT_EQ(StructFieldNames(schemas_array), + (std::vector{"schema_id", "fields", "partition_keys", "primary_keys", + "options", "comment", "update_time"})); + ASSERT_EQ(schemas_array->length(), 1); + auto schema_id_array = std::dynamic_pointer_cast(schemas_array->field(0)); + auto primary_keys_array = + std::dynamic_pointer_cast(schemas_array->field(3)); + auto update_time_array = + std::dynamic_pointer_cast(schemas_array->field(6)); + ASSERT_TRUE(schema_id_array); + ASSERT_TRUE(primary_keys_array); + ASSERT_TRUE(update_time_array); + ASSERT_EQ(schema_id_array->Value(0), 0); + ASSERT_EQ(primary_keys_array->GetString(0), R"(["pk"])"); + + ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options)); + auto branches_array = SingleStructChunk(branches_result); + ASSERT_EQ(StructFieldNames(branches_array), + (std::vector{"branch_name", "create_time"})); + ASSERT_EQ(branches_array->length(), 1); + auto branch_name_array = + std::dynamic_pointer_cast(branches_array->field(0)); + ASSERT_TRUE(branch_name_array); + ASSERT_EQ(branch_name_array->GetString(0), "main"); + auto branch_create_time_array = + std::dynamic_pointer_cast(branches_array->field(1)); + ASSERT_TRUE(branch_create_time_array); + + ASSERT_OK_AND_ASSIGN(auto manifests_result, + ReadSystemTable(table_path + "$manifests", options)); + auto manifests_array = SingleStructChunk(manifests_result); + ASSERT_EQ(StructFieldNames(manifests_array), + (std::vector{"file_name", "file_size", "num_added_files", + "num_deleted_files", "schema_id", "min_partition_stats", + "max_partition_stats", "min_row_id", "max_row_id"})); + ASSERT_GT(manifests_array->length(), 0); + auto manifest_file_name_array = + std::dynamic_pointer_cast(manifests_array->field(0)); + auto manifest_file_size_array = + std::dynamic_pointer_cast(manifests_array->field(1)); + auto manifest_num_added_files_array = + std::dynamic_pointer_cast(manifests_array->field(2)); + auto manifest_schema_id_array = + std::dynamic_pointer_cast(manifests_array->field(4)); + ASSERT_TRUE(manifest_file_name_array); + ASSERT_TRUE(manifest_file_size_array); + ASSERT_TRUE(manifest_num_added_files_array); + ASSERT_TRUE(manifest_schema_id_array); + ASSERT_EQ(manifest_file_name_array->GetString(0).find("manifest-"), 0); + ASSERT_GT(manifest_file_size_array->Value(0), 0); + ASSERT_GE(manifest_num_added_files_array->Value(0), 1); + ASSERT_EQ(manifest_schema_id_array->Value(0), 0); + + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(StructFieldNames(files_array), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_GT(files_array->length(), 0); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto bucket_array = std::dynamic_pointer_cast(files_array->field(1)); + auto file_path_array = std::dynamic_pointer_cast(files_array->field(2)); + auto file_format_array = std::dynamic_pointer_cast(files_array->field(3)); + auto file_schema_id_array = std::dynamic_pointer_cast(files_array->field(4)); + auto record_count_array = std::dynamic_pointer_cast(files_array->field(6)); + auto file_size_array = std::dynamic_pointer_cast(files_array->field(7)); + auto min_sequence_number_array = + std::dynamic_pointer_cast(files_array->field(13)); + auto max_sequence_number_array = + std::dynamic_pointer_cast(files_array->field(14)); + auto creation_time_array = + std::dynamic_pointer_cast(files_array->field(15)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(bucket_array); + ASSERT_TRUE(file_path_array); + ASSERT_TRUE(file_format_array); + ASSERT_TRUE(file_schema_id_array); + ASSERT_TRUE(record_count_array); + ASSERT_TRUE(file_size_array); + ASSERT_TRUE(min_sequence_number_array); + ASSERT_TRUE(max_sequence_number_array); + ASSERT_TRUE(creation_time_array); + ASSERT_TRUE(partition_array->IsNull(0)); + ASSERT_EQ(bucket_array->Value(0), 0); + ASSERT_NE(file_path_array->GetString(0).find("/bucket-0/"), std::string::npos); + ASSERT_EQ(file_format_array->GetString(0), "parquet"); + ASSERT_EQ(file_schema_id_array->Value(0), 0); + ASSERT_EQ(record_count_array->Value(0), 1); + ASSERT_GT(file_size_array->Value(0), 0); + ASSERT_GE(min_sequence_number_array->Value(0), 0); + ASSERT_GE(max_sequence_number_array->Value(0), min_sequence_number_array->Value(0)); + ASSERT_FALSE(creation_time_array->IsNull(0)); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::NUM_LEVELS, "3"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_1), /*commit_identifier=*/0, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult compacted_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected_compacted; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected_compacted) + .ok()); + ASSERT_TRUE(compacted_result.array->Equals(expected_compacted)) + << compacted_result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + TestHelper::MakeRecordBatch(row_type, R"([[1, 11], [3, 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(SystemTableReadResult stale_result, + ReadSystemTable(table_path + "$ro", options)); + ASSERT_TRUE(stale_result.array->Equals(expected_compacted)) << stale_result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_3, + TestHelper::MakeRecordBatch(row_type, R"([[2, 21], [3, 31]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_3), /*commit_identifier=*/2, helper.get())); + ASSERT_OK_AND_ASSIGN(SystemTableReadResult refreshed_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_refreshed; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 11], [0, 2, 21], [0, 3, 31]])"}, &expected_refreshed) + .ok()); + ASSERT_TRUE(refreshed_result.array->Equals(expected_refreshed)) + << refreshed_result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedAppendOnlySystemTableWithStreamingScan) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$ro", options, /*streaming_mode=*/true)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicatePushdown) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + arrow::field("extra", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10, 100], [2, 20, 200], [3, 30, 300]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_1), /*commit_identifier=*/0, helper.get())); + + std::shared_ptr predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + + ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); + ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, + ro_scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, + TableScan::Create(std::move(ro_scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); + ASSERT_EQ(ro_file_count, 1); + + ReadContextBuilder read_context_builder(table_path + "$ro"); + // Do not enable row-level predicate filtering: the result must come from the file reader's + // row-group/page predicate pushdown. + read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(ro_plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(expected_type, {R"([[0, 2, 20]])"}, + &expected) + .ok()); + ASSERT_TRUE(result->Equals(expected)) << result->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { + auto payload_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("payload", payload_type), + arrow::field("extra", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[1, [10, "x"], 100], [2, [20, "y"], 200]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + + ScanContextBuilder scan_context_builder(table_path + "$ro"); + scan_context_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); + + auto projected_schema = arrow::schema({ + arrow::field("k", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int32())})), + }); + auto c_projected_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); + ReadContextBuilder read_context_builder(table_path + "$ro"); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + + std::shared_ptr expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int32())})), + }); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, [10]], [0, 2, [20]]])"}, &expected) + .ok()); + ASSERT_TRUE(result->Equals(expected)) << result->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithBranch) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::NUM_LEVELS, "3"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_batch, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(branch_batch), /*commit_identifier=*/0, helper.get())); + + std::string branch_path = PathUtil::JoinPath(table_path, "branch/branch-rt"); + std::filesystem::create_directories(branch_path); + ASSERT_TRUE(TestUtil::CopyDirectory(PathUtil::JoinPath(table_path, "schema"), + PathUtil::JoinPath(branch_path, "schema"))); + ASSERT_TRUE(TestUtil::CopyDirectory(PathUtil::JoinPath(table_path, "snapshot"), + PathUtil::JoinPath(branch_path, "snapshot"))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_batch, + TestHelper::MakeRecordBatch(row_type, R"([[1, 11], [3, 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(main_batch), /*commit_identifier=*/1, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$branch_rt$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult main_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_main; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 11], [0, 2, 20], [0, 3, 30]])"}, &expected_main) + .ok()); + ASSERT_TRUE(main_result.array->Equals(expected_main)) << main_result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithFirstRowMergeEngine) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}, {Options::NUM_LEVELS, "5"}, + {Options::MERGE_ENGINE, "first-row"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true, + /*ignore_if_exists=*/false, PathUtil::JoinPath(dir->Str(), "tmp"))); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{"dt", "pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["20260527", "a", 1]])", + /*partition_map=*/{{"dt", "20260527"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto file_path_array = std::dynamic_pointer_cast(files_array->field(2)); + auto min_key_array = std::dynamic_pointer_cast(files_array->field(8)); + auto max_key_array = std::dynamic_pointer_cast(files_array->field(9)); + auto null_value_counts_array = + std::dynamic_pointer_cast(files_array->field(10)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + auto max_value_stats_array = + std::dynamic_pointer_cast(files_array->field(12)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(file_path_array); + ASSERT_TRUE(min_key_array); + ASSERT_TRUE(max_key_array); + ASSERT_TRUE(null_value_counts_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_TRUE(max_value_stats_array); + ASSERT_EQ(partition_array->GetString(0), "{20260527}"); + ASSERT_NE(file_path_array->GetString(0).find("/dt=20260527/bucket-0/"), std::string::npos); + ASSERT_EQ(min_key_array->GetString(0), "[a]"); + ASSERT_EQ(max_key_array->GetString(0), "[a]"); + ASSERT_EQ(null_value_counts_array->GetString(0), "{dt=0, pk=0, v=0}"); + ASSERT_EQ(min_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}"); + ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}"); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite) { + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("id", arrow::int32()), + arrow::field("score", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + }; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + WriteContextBuilder write_context_builder(table_path, "partial-write"); + write_context_builder.WithWriteSchema({"score"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write, + FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])", + /*partition_map=*/{{"dt", "20260724"}}, /*bucket=*/0, {})); + ASSERT_OK(write->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(std::vector> commit_messages, + write->PrepareCommit()); + ASSERT_OK(write->Close()); + + CommitContextBuilder commit_context_builder(table_path, "partial-write"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Commit(commit_messages)); + + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto null_value_counts_array = + std::dynamic_pointer_cast(files_array->field(10)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + auto max_value_stats_array = + std::dynamic_pointer_cast(files_array->field(12)); + auto write_cols_array = std::dynamic_pointer_cast(files_array->field(19)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(null_value_counts_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_TRUE(max_value_stats_array); + ASSERT_TRUE(write_cols_array); + + ASSERT_EQ(partition_array->GetString(0), "{20260724}"); + ASSERT_EQ(null_value_counts_array->GetString(0), "{dt=0, id=2, score=0}"); + ASSERT_EQ(min_value_stats_array->GetString(0), "{dt=20260724, id=null, score=10}"); + ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260724, id=null, score=20}"); + auto write_cols_values = + std::dynamic_pointer_cast(write_cols_array->values()); + ASSERT_TRUE(write_cols_values); + ASSERT_EQ(write_cols_values->length(), 1); + ASSERT_EQ(write_cols_values->GetString(0), "score"); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) { + arrow::FieldVector fields = { + arrow::field("dt", arrow::date32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[10440, 1]])", + /*partition_map=*/{{"dt", "1998-08-02"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + ASSERT_TRUE(partition_array); + ASSERT_EQ(partition_array->GetString(0), "{1998-08-02}"); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableWithSchemaEvolutionStats) { + std::map options = {{Options::FILE_SYSTEM, "local"}}; + std::string table_path = paimon::test::GetDataDir() + + "/orc/append_table_with_alter_table_with_dense_field.db/" + "append_table_with_alter_table_with_dense_field"; + + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(StructFieldNames(files_array), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_GT(files_array->length(), 0); + + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto schema_id_array = std::dynamic_pointer_cast(files_array->field(4)); + auto null_value_counts_array = + std::dynamic_pointer_cast(files_array->field(10)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + auto max_value_stats_array = + std::dynamic_pointer_cast(files_array->field(12)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(schema_id_array); + ASSERT_TRUE(null_value_counts_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_TRUE(max_value_stats_array); + + bool found_old_schema_file = false; + bool found_latest_schema_file = false; + for (int64_t i = 0; i < files_array->length(); ++i) { + std::string partition = partition_array->GetString(i); + ASSERT_TRUE(partition == "{0}" || partition == "{1}"); + + std::string null_value_counts = null_value_counts_array->GetString(i); + std::string min_value_stats = min_value_stats_array->GetString(i); + std::string max_value_stats = max_value_stats_array->GetString(i); + ASSERT_NE(null_value_counts.find("f4="), std::string::npos); + ASSERT_NE(min_value_stats.find("f4="), std::string::npos); + ASSERT_NE(max_value_stats.find("f4="), std::string::npos); + ASSERT_EQ(null_value_counts.find("f0="), std::string::npos); + ASSERT_EQ(min_value_stats.find("f0="), std::string::npos); + ASSERT_EQ(max_value_stats.find("f0="), std::string::npos); + + if (schema_id_array->Value(i) == 0) { + found_old_schema_file = true; + ASSERT_NE(null_value_counts.find("f4="), std::string::npos); + ASSERT_NE(min_value_stats.find("f4=null"), std::string::npos); + ASSERT_NE(max_value_stats.find("f4=null"), std::string::npos); + } else if (schema_id_array->Value(i) == 1) { + found_latest_schema_file = true; + } + } + ASSERT_TRUE(found_old_schema_file); + ASSERT_TRUE(found_latest_schema_file); +} + +TEST(SystemTableReadInteTest, TestReadManifestAndFilesSystemTablesForEmptyTable) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, + catalog->GetTableLocation(Identifier("db1", "tbl1"))); + ASSERT_OK_AND_ASSIGN(auto manifests_result, + ReadSystemTable(table_path + "$manifests", options)); + ASSERT_EQ(manifests_result.array, nullptr); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + ASSERT_EQ(files_result.array, nullptr); +} + +TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { + const char* old_tz = std::getenv("TZ"); + std::optional old_timezone; + if (old_tz != nullptr) { + old_timezone = old_tz; + } + setenv("TZ", "Asia/Shanghai", /*overwrite=*/1); + tzset(); + ScopeGuard timezone_guard([old_timezone]() { + if (old_timezone) { + setenv("TZ", old_timezone->c_str(), /*overwrite=*/1); + } else { + unsetenv("TZ"); + } + tzset(); + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string source_path = + GetDataDir() + "/parquet/append_table_with_rt_branch.db/append_table_with_rt_branch"; + std::string table_path = PathUtil::JoinPath(dir->Str(), "metadata_table"); + ASSERT_TRUE(TestUtil::CopyDirectory(std::filesystem::path(source_path), + std::filesystem::path(table_path))); + + auto fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(Tag tag, + Tag::FromPath(fs, GetDataDir() + "/orc/append_table_with_tag.db/" + "append_table_with_tag/tag/tag-1")); + ASSERT_OK_AND_ASSIGN(std::string tag_json, tag.ToJsonString()); + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(table_path, "tag"))); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "tag/tag-release"), tag_json, + /*overwrite=*/true)); + + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(table_path, "consumer"))); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "consumer/consumer-c1"), + R"({"nextSnapshot":3})", + /*overwrite=*/true)); + + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}}; + + ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options)); + auto branches_array = SingleStructChunk(branches_result); + ASSERT_EQ(branches_array->length(), 2); + auto branch_name_array = + std::dynamic_pointer_cast(branches_array->field(0)); + ASSERT_TRUE(branch_name_array); + ASSERT_EQ(branch_name_array->GetString(0), "main"); + ASSERT_EQ(branch_name_array->GetString(1), "rt"); + + ASSERT_OK_AND_ASSIGN(auto tags_result, ReadSystemTable(table_path + "$tags", options)); + auto tags_array = SingleStructChunk(tags_result); + ASSERT_EQ(StructFieldNames(tags_array), + (std::vector{"tag_name", "snapshot_id", "schema_id", "commit_time", + "record_count", "create_time", "time_retained"})); + ASSERT_EQ(tags_array->length(), 1); + auto tag_name_array = std::dynamic_pointer_cast(tags_array->field(0)); + auto tag_snapshot_array = std::dynamic_pointer_cast(tags_array->field(1)); + auto tag_commit_time_array = + std::dynamic_pointer_cast(tags_array->field(3)); + auto tag_record_count_array = + std::dynamic_pointer_cast(tags_array->field(4)); + auto tag_create_time_array = + std::dynamic_pointer_cast(tags_array->field(5)); + auto tag_time_retained_array = + std::dynamic_pointer_cast(tags_array->field(6)); + ASSERT_TRUE(tag_name_array); + ASSERT_TRUE(tag_snapshot_array); + ASSERT_TRUE(tag_commit_time_array); + ASSERT_TRUE(tag_record_count_array); + ASSERT_TRUE(tag_create_time_array); + ASSERT_TRUE(tag_time_retained_array); + ASSERT_EQ(tag_name_array->GetString(0), "release"); + ASSERT_EQ(tag_snapshot_array->Value(0), tag.Id()); + ASSERT_OK_AND_ASSIGN( + Timestamp tag_commit_time, + DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(tag.TimeMillis()))); + ASSERT_EQ(tag_commit_time_array->Value(0), tag_commit_time.GetMillisecond()); + ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount()); + ASSERT_FALSE(tag_create_time_array->IsNull(0)); + ASSERT_EQ(tag_create_time_array->Value(0), 1770185290000); + ASSERT_EQ(tag_time_retained_array->GetString(0), "3.000000"); + + ASSERT_OK_AND_ASSIGN(auto consumers_result, + ReadSystemTable(table_path + "$consumers", options)); + auto consumers_array = SingleStructChunk(consumers_result); + ASSERT_EQ(StructFieldNames(consumers_array), + (std::vector{"consumer_id", "next_snapshot_id"})); + ASSERT_EQ(consumers_array->length(), 1); + auto consumer_id_array = + std::dynamic_pointer_cast(consumers_array->field(0)); + auto next_snapshot_array = + std::dynamic_pointer_cast(consumers_array->field(1)); + ASSERT_TRUE(consumer_id_array); + ASSERT_TRUE(next_snapshot_array); + ASSERT_EQ(consumer_id_array->GetString(0), "c1"); + ASSERT_EQ(next_snapshot_array->Value(0), 3); +} + TEST(SystemTableReadInteTest, TestReadAuditLogSystemTable) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), @@ -784,6 +1691,67 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithChangelogR ])"); } +TEST(SystemTableReadInteTest, TestStreamingBinlogPacksUpdateBeforeAndAfter) { + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, + {Options::WRITE_BUFFER_SIZE, "1"}, {Options::WRITE_BUFFER_SPILLABLE, "false"}, + }; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true)); + + std::vector row_kinds_1 = { + RecordBatch::RowKind::INSERT, + RecordBatch::RowKind::UPDATE_BEFORE, + }; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1], ["b", 2]])", + /*partition_map=*/{}, /*bucket=*/0, row_kinds_1)); + std::vector row_kinds_2 = { + RecordBatch::RowKind::UPDATE_AFTER, + RecordBatch::RowKind::DELETE, + }; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["b", 3], ["c", 4]])", + /*partition_map=*/{}, /*bucket=*/0, row_kinds_2)); + std::vector> batches; + batches.push_back(std::move(batch_1)); + batches.push_back(std::move(batch_2)); + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map streaming_options = options; + streaming_options[Options::SCAN_MODE] = "from-snapshot"; + streaming_options[Options::SCAN_SNAPSHOT_ID] = "1"; + ASSERT_OK_AND_ASSIGN( + auto result, + ReadSystemTable(PathUtil::JoinPath(dir->Str(), "foo.db/bar$binlog"), streaming_options, + /*streaming_mode=*/true, /*predicate=*/nullptr, + /*read_field_names=*/{}, /*read_next_plan=*/true)); + ASSERT_TRUE(result.array); + ASSERT_EQ(result.array->num_chunks(), 2); + auto array = std::dynamic_pointer_cast( + arrow::Concatenate(result.array->chunks()).ValueOrDie()); + ASSERT_TRUE(array); + ASSERT_EQ(StructFieldNames(array), (std::vector{"rowkind", "pk", "v"})); + AssertStructArrayEqualsJson(array, R"([ + ["+I", ["a"], [1]], + ["+U", ["b", "b"], [2, 3]], + ["-D", ["c"], [4]] + ])"); +} + TEST(SystemTableReadInteTest, TestReadBinlogSystemTableWithNullValue) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8(), /*nullable=*/false), @@ -910,7 +1878,7 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") @@ -990,7 +1958,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .SetPredicate(predicate) @@ -1094,7 +2062,7 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { "/append_complex_data.db/append_complex_data"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f6", "f2", "f4", "f3", "f5"}); + context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -1167,7 +2135,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") .AddOption("test.enable-adaptive-prefetch-strategy", @@ -1243,7 +2211,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") .AddOption("test.enable-adaptive-prefetch-strategy", @@ -1328,7 +2296,7 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") @@ -1572,7 +2540,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f0", "f3", "f1"}); + context_builder.SetReadFieldNames({"f0", "f3", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -1746,7 +2714,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { "/append_table_with_alter_table.db/append_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"a", "k", "key1", "d", "key0", "c"}); + context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -1825,7 +2793,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) "append_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"a", "k", "key1", "d", "key0", "c"}); + context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -1898,7 +2866,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -1983,7 +2951,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2067,7 +3035,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); + context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPrefetchCacheMode(param.cache_mode); @@ -2150,7 +3118,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -2242,7 +3210,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key0", "key1", "k", "c", "d", "a", "e"}); + context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2360,7 +3328,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { "append_table_alter_table_with_cast/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); + context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2441,7 +3409,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); + context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); @@ -2759,7 +3727,8 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { TEST_P(ReadInteTest, TestSpecificFs) { class CountableInputStream : public InputStream { public: - CountableInputStream(const std::shared_ptr& input, size_t* io_count) + CountableInputStream(const std::shared_ptr& input, + std::atomic* io_count) : input_(input), io_count_(io_count) {} ~CountableInputStream() override = default; @@ -2769,17 +3738,17 @@ TEST_P(ReadInteTest, TestSpecificFs) { Result GetPos() const override { return input_->GetPos(); } - Result Read(char* buffer, uint32_t size) override { - (*io_count_)++; + Result Read(char* buffer, int64_t size) override { + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size); } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { - (*io_count_)++; + Result Read(char* buffer, int64_t size, int64_t offset) override { + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size, offset); } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->ReadAsync(buffer, size, offset, std::move(callback)); } @@ -2789,17 +3758,17 @@ TEST_P(ReadInteTest, TestSpecificFs) { Result GetUri() const override { return input_->GetUri(); } - Result Length() const override { + Result Length() const override { return input_->Length(); } std::shared_ptr input_; - size_t* io_count_; + std::atomic* io_count_; }; class CountableFileSystem : public FileSystem { public: - CountableFileSystem(const std::shared_ptr& fs, size_t* io_count) + CountableFileSystem(const std::shared_ptr& fs, std::atomic* io_count) : fs_(fs), io_count_(io_count) {} ~CountableFileSystem() override = default; @@ -2837,10 +3806,10 @@ TEST_P(ReadInteTest, TestSpecificFs) { } std::shared_ptr fs_; - size_t* io_count_; + std::atomic* io_count_; }; - size_t io_count = 0; + std::atomic io_count = {0}; auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; @@ -2893,7 +3862,439 @@ TEST_P(ReadInteTest, TestSpecificFs) { &expected_array); ASSERT_TRUE(array_status.ok()); ASSERT_TRUE(result_array->Equals(expected_array)); - ASSERT_GT(io_count, 0); + ASSERT_GT(io_count.load(std::memory_order_relaxed), 0); +} + +// ============================================================================= +// Global System Table Tests +// ============================================================================= + +namespace { + +Result ReadGlobalSystemTable( + const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, + const std::string& warehouse, const std::map& options) { + GlobalSystemTableContext ctx; + ctx.catalog = catalog; + ctx.fs = fs; + ctx.warehouse = warehouse; + ctx.catalog_options = options; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(table_name, ctx)); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + + std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); + + ScanContextBuilder scan_context_builder(sys_path); + scan_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + system_table->NewScan(scan_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(sys_path); + read_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + system_table->NewRead(read_context)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + return SystemTableReadResult(std::move(batch_reader), result); +} + +} // namespace + +TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { + std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {CatalogOptionsSystemTable::kEnabledOption, "true"}, + {"custom.catalog.option", "test-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("catalog_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto value_array = std::dynamic_pointer_cast(struct_array->field(1)); + ASSERT_TRUE(key_array); + ASSERT_TRUE(value_array); + + // Build a map from the result + std::map result_map; + for (int64_t i = 0; i < struct_array->length(); ++i) { + result_map[key_array->GetString(i)] = value_array->GetString(i); + } + ASSERT_EQ(result_map["file-system"], "local"); + ASSERT_EQ(result_map["file.format"], "orc"); +} + +TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {"table.option.custom", "my-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + // Create a database and table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // Verify basic enumeration works + ASSERT_OK_AND_ASSIGN(auto dbs, catalog->ListDatabases()); + ASSERT_TRUE(std::find(dbs.begin(), dbs.end(), "test_db") != dbs.end()); + ASSERT_OK_AND_ASSIGN(auto tbls, catalog->ListTables("test_db")); + ASSERT_TRUE(std::find(tbls.begin(), tbls.end(), "test_tbl") != tbls.end()); + + // Verify schema loads and has Options + ASSERT_OK_AND_ASSIGN(auto loaded_schema, + catalog->LoadTableSchema(Identifier("test_db", "test_tbl"))); + auto ds = std::dynamic_pointer_cast(loaded_schema); + ASSERT_TRUE(ds != nullptr) << "LoadTableSchema did not return DataSchema"; + ASSERT_FALSE(ds->Options().empty()) << "Table schema has no options"; + + // Directly test BuildRows + { + GlobalSystemTableContext ctx; + ctx.catalog = catalog.get(); + ctx.fs = catalog->GetFileSystem(); + ctx.warehouse = warehouse; + ctx.catalog_options = options; + AllTableOptionsSystemTable table(ctx); + ASSERT_OK_AND_ASSIGN(auto rows, table.BuildRows()); + ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; + } + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("all_table_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto key_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto val_array = std::dynamic_pointer_cast(struct_array->field(3)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(key_array); + ASSERT_TRUE(val_array); + + // Verify that our table's options appear in the result + bool found_db = false; + bool found_format = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + auto db_name = std::string(db_array->GetString(i)); + auto tbl_name = std::string(tbl_array->GetString(i)); + if (db_name == "test_db" && tbl_name == "test_tbl") { + found_db = true; + auto key_str = std::string(key_array->GetString(i)); + if (key_str == "file.format") { + EXPECT_EQ(std::string(val_array->GetString(i)), "orc"); + found_format = true; + } + } + } + ASSERT_TRUE(found_db) << "test_db.test_tbl not found in sys.all_table_options"; + ASSERT_TRUE(found_format) << "file.format option not found in sys.all_table_options"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalTables) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {"owner", "alice"}, + {"createdAt", "1000"}, + {"createdBy", "creator"}, + {"updatedAt", "2000"}, + {"updatedBy", "updater"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + + // Create a database and a PK table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({ + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, + catalog->GetTableLocation(Identifier("test_db", "test_tbl"))); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(typed_schema->fields()), R"([["k", 1]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map no_pk_options = options; + no_pk_options[Options::BUCKET_KEY] = "pk"; + ::ArrowSchema no_pk_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &no_pk_schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_no_pk_tbl"), &no_pk_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, no_pk_options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&no_pk_schema); + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1); + ASSERT_EQ(StructFieldNames(struct_array), + (std::vector{ + "database_name", "table_name", "table_type", "partitioned", "primary_key", + "owner", "created_at", "created_by", "updated_at", "updated_by", "record_count", + "file_size_in_bytes", "file_count", "last_file_creation_time"})); + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto type_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto part_array = std::dynamic_pointer_cast(struct_array->field(3)); + auto pk_array = std::dynamic_pointer_cast(struct_array->field(4)); + auto owner_array = std::dynamic_pointer_cast(struct_array->field(5)); + auto created_at_array = std::dynamic_pointer_cast(struct_array->field(6)); + auto created_by_array = std::dynamic_pointer_cast(struct_array->field(7)); + auto updated_at_array = std::dynamic_pointer_cast(struct_array->field(8)); + auto updated_by_array = std::dynamic_pointer_cast(struct_array->field(9)); + auto record_count_array = std::dynamic_pointer_cast(struct_array->field(10)); + auto file_size_array = std::dynamic_pointer_cast(struct_array->field(11)); + auto file_count_array = std::dynamic_pointer_cast(struct_array->field(12)); + auto last_creation_time_array = + std::dynamic_pointer_cast(struct_array->field(13)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(type_array); + ASSERT_TRUE(part_array); + ASSERT_TRUE(pk_array); + ASSERT_TRUE(owner_array); + ASSERT_TRUE(created_at_array); + ASSERT_TRUE(created_by_array); + ASSERT_TRUE(updated_at_array); + ASSERT_TRUE(updated_by_array); + ASSERT_TRUE(record_count_array); + ASSERT_TRUE(file_size_array); + ASSERT_TRUE(file_count_array); + ASSERT_TRUE(last_creation_time_array); + + // Find our table by table name + bool found = false; + bool found_no_pk = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + if (std::string(tbl_array->GetString(i)) == "test_tbl") { + EXPECT_EQ(std::string(db_array->GetString(i)), "test_db"); + EXPECT_EQ(std::string(type_array->GetString(i)), "table"); + EXPECT_FALSE(part_array->Value(i)); + EXPECT_TRUE(pk_array->Value(i)); + EXPECT_EQ(owner_array->GetString(i), "alice"); + EXPECT_EQ(created_at_array->Value(i), 1000); + EXPECT_EQ(created_by_array->GetString(i), "creator"); + EXPECT_EQ(updated_at_array->Value(i), 2000); + EXPECT_EQ(updated_by_array->GetString(i), "updater"); + EXPECT_TRUE(record_count_array->IsNull(i)); + EXPECT_TRUE(file_size_array->IsNull(i)); + EXPECT_TRUE(file_count_array->IsNull(i)); + EXPECT_TRUE(last_creation_time_array->IsNull(i)); + found = true; + } else if (std::string(tbl_array->GetString(i)) == "test_no_pk_tbl") { + EXPECT_FALSE(pk_array->Value(i)); + found_no_pk = true; + } + } + ASSERT_TRUE(found) << "table not found in sys.tables"; + ASSERT_TRUE(found_no_pk) << "no-PK table not found in sys.tables"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("region", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt", "region"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-13", "cn", 1]])", + /*partition_map=*/{{"dt", "2026-07-13"}, {"region", "cn"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, + warehouse, options)); + auto struct_array = SingleStructChunk(part_result); + ASSERT_TRUE(struct_array); + ASSERT_EQ(StructFieldNames(struct_array), + (std::vector{"database_name", "table_name", "partition_name", + "record_count", "file_size_in_bytes", "file_count", + "last_file_creation_time", "done"})); + ASSERT_EQ(struct_array->length(), 1); + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto table_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto partition_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto creation_time_array = std::dynamic_pointer_cast(struct_array->field(6)); + auto done_array = std::dynamic_pointer_cast(struct_array->field(7)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(table_array); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(creation_time_array); + ASSERT_TRUE(done_array); + EXPECT_EQ(db_array->GetString(0), "foo"); + EXPECT_EQ(table_array->GetString(0), "bar"); + EXPECT_EQ(partition_array->GetString(0), "dt=2026-07-13/region=cn"); + EXPECT_FALSE(creation_time_array->IsNull(0)); + EXPECT_FALSE(done_array->Value(0)); +} + +TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("v", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + Identifier identifier("test_db", "test_tbl"); + ASSERT_OK(catalog->CreateTable(identifier, &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog->GetTableLocation(identifier)); + std::shared_ptr fs = catalog->GetFileSystem(); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "schema/schema-0"), "{invalid-json", + /*overwrite=*/true)); + + for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + } +} + +TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-19", 1]])", + /*partition_map=*/{{"dt", "2026-07-19"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + std::shared_ptr fs = catalog->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(warehouse, "foo.db/bar"); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", + /*overwrite=*/true)); + + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); +} + +TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-19", 1]])", + /*partition_map=*/{{"dt", "2026-07-19"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + std::shared_ptr fs = catalog->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(warehouse, "foo.db/bar"); + SnapshotManager snapshot_manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_OK(fs->WriteFile( + PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", + /*overwrite=*/true)); + + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); } } // namespace paimon::test diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index ad4c66fe..a9294ef0 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" @@ -40,6 +42,7 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" +#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" @@ -53,6 +56,7 @@ #include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/plan.h" +#include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/io_exception_helper.h" @@ -66,8 +70,10 @@ class DataSplit; } // namespace paimon namespace paimon::test { +using ScanAndReadParam = std::tuple; + class ScanAndReadInteTest : public testing::Test, - public ::testing::WithParamInterface> { + public ::testing::WithParamInterface { public: void CheckStreamScanResult( const std::unique_ptr& table_scan, const std::unique_ptr& table_read, @@ -130,13 +136,36 @@ class ScanAndReadInteTest : public testing::Test, } void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - auto [file_format, enable_prefetch] = GetParam(); read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); - if (enable_prefetch) { + if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } + Result> FinishScanContext(ScanContextBuilder& builder) { + if (EnableSnapshotLiveManifestCache()) { + if (!snapshot_live_manifest_cache_) { + snapshot_live_manifest_cache_ = + std::make_shared(/*max_weight=*/64 * 1024 * 1024); + } + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(snapshot_live_manifest_cache_); + } + return builder.Finish(); + } + + std::string FileFormat() const { + return std::get<0>(GetParam()); + } + + bool EnablePrefetch() const { + return std::get<1>(GetParam()); + } + + bool EnableSnapshotLiveManifestCache() const { + return std::get<2>(GetParam()); + } + void AdjustSplitWithExternalPath(const std::string& src_path, const std::string& target_path, bool adjust_index, std::vector>* splits_ptr) { @@ -166,6 +195,8 @@ class ScanAndReadInteTest : public testing::Test, } private: + std::shared_ptr snapshot_live_manifest_cache_; + std::shared_ptr arrow_data_type_ = std::dynamic_pointer_cast(DataField::ConvertDataFieldsToArrowStructType( {SpecialFields::ValueKind(), DataField(0, arrow::field("f0", arrow::utf8())), @@ -174,21 +205,24 @@ class ScanAndReadInteTest : public testing::Test, DataField(3, arrow::field("f3", arrow::float64()))})); }; -std::vector> GetTestValuesForScanAndReadInteTest() { - std::vector> values = {{"parquet", false}, {"parquet", true}}; +std::vector GetTestValuesForScanAndReadInteTest() { + std::vector values; + for (bool enable_snapshot_live_manifest_cache : {false, true}) { + values.emplace_back("parquet", false, enable_snapshot_live_manifest_cache); + values.emplace_back("parquet", true, enable_snapshot_live_manifest_cache); #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc", false); - values.emplace_back("orc", true); + values.emplace_back("orc", false, enable_snapshot_live_manifest_cache); + values.emplace_back("orc", true, enable_snapshot_live_manifest_cache); #endif + } return values; } INSTANTIATE_TEST_SUITE_P(FileFormatAndEnablePaimonPrefetch, ScanAndReadInteTest, - ::testing::ValuesIn(std::vector>( - GetTestValuesForScanAndReadInteTest()))); + ::testing::ValuesIn(GetTestValuesForScanAndReadInteTest())); TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; bool run_complete = false; @@ -199,7 +233,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = scan_context_builder.Finish(); + Result> scan_context = FinishScanContext(scan_context_builder); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = TableScan::Create(std::move(scan_context).value()); @@ -243,7 +277,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { } TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -255,7 +289,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = scan_context_builder.Finish(); + Result> scan_context = FinishScanContext(scan_context_builder); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = TableScan::Create(std::move(scan_context).value()); @@ -301,13 +335,13 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -335,16 +369,20 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -380,13 +418,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -420,16 +458,20 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -472,13 +514,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { } TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -509,14 +551,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { } TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -562,30 +604,29 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { } // test pk with dv -TEST_F(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { - auto check_result = [&](const std::string& file_format) { - std::string table_path = GetDataDir() + "/" + file_format + - "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; +TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { + auto file_format = FileFormat(); + std::string table_path = GetDataDir() + "/" + file_format + + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - // normal batch scan case for pk+dv, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + // normal batch scan case for pk+dv, all data in level 0 is filtered out + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - ReadContextBuilder read_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ReadContextBuilder read_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 6); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(batch_reader.get())); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 6); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - // check result - auto expected = std::make_shared( - arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ + // check result + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ [0, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 10, 1, 11.0], [0, "Alice", 10, 1, 19.1], [0, "Alex", 10, 0, 16.1], @@ -595,23 +636,23 @@ TEST_F(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { [0, "Lucy", 20, 1, 14.1], [0, "Paul", 20, 1, 18.1] ])") - .ValueOrDie()); - ASSERT_TRUE(expected); - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - }; - for (auto [file_format, enable_prefetch] : GetTestValuesForScanAndReadInteTest()) { - check_result(file_format); - } - check_result("avro"); + .ValueOrDie()); + ASSERT_TRUE(expected); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + // CountRows should match the number of visible rows returned by CreateReader. + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -627,7 +668,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBucketFilter) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); // all data in level 0 & not in partition 10, bucket 1 is filtered out std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -635,7 +676,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -662,7 +703,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level @@ -678,7 +719,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -704,7 +745,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -715,7 +756,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -724,7 +765,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -732,7 +773,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // data in partition 20 is truncated ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -761,7 +802,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -769,7 +810,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -813,7 +854,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -821,7 +862,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -859,13 +900,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -896,7 +937,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -904,7 +945,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -955,7 +996,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -963,7 +1004,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") .AddOption(Options::SCAN_MODE, "from-snapshot") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1000,19 +1041,19 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { } TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN( auto read_context, - read_context_builder.SetReadSchema({"shopId", "dt", "hr", "col0", "col1", "col2"}) + read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); @@ -1052,13 +1093,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { // test pk with mor TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1068,7 +1109,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + auto splits = result_plan->Splits(); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); // check result @@ -1089,10 +1131,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1100,7 +1146,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // with merge read ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1110,7 +1156,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + auto splits = result_plan->Splits(); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); // check result @@ -1128,10 +1175,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndBucketFilter) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1139,7 +1190,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1169,7 +1220,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1195,7 +1246,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1229,7 +1280,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1241,7 +1292,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1250,7 +1301,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -1259,7 +1310,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho .AddOption(Options::MERGE_ENGINE, "aggregation") .AddOption("fields.f3.aggregate-function", "rbm32"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1289,7 +1340,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho } TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1298,7 +1349,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh .AddOption(Options::MERGE_ENGINE, "aggregation") .AddOption("fields.f3.aggregate-function", "rbm32"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1316,7 +1367,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh } TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1329,7 +1380,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1359,7 +1410,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica } TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1372,7 +1423,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1402,7 +1453,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1410,7 +1461,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // merging ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1443,7 +1494,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1451,7 +1502,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1488,7 +1539,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1496,7 +1547,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1547,7 +1598,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1557,7 +1608,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") .AddOption(Options::SCAN_MODE, "from-snapshot") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1594,7 +1645,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1605,7 +1656,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1632,7 +1683,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { // test first row merge engine TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; @@ -1640,7 +1691,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { // normal batch scan case for pk+first row, all data in level 0 is filtered out ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1672,7 +1723,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; @@ -1681,7 +1732,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1725,14 +1776,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { } TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1765,13 +1816,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1793,7 +1844,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { [0, "Emily", 10, 0, 13.1], [0, "Alice", 10, 1, 21.1], [0, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 10, 1, 11.0], -[0, "Whether I shall turn out to be the hero of my own life.", 10, 1, 19.1] +[0, "Whether I shall turn out to be the hero of my own life.", 10, 1, 19.1], +[0, "Lucy", 20, 1, 14.1], +[0, "Paul", 20, 1, 18.1] ])") .ValueOrDie()); ASSERT_TRUE(expected); @@ -1801,7 +1854,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { } TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); auto check_result = [&](const std::string& table_path, @@ -1809,7 +1862,7 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { const std::shared_ptr& expected) { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1868,7 +1921,7 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { } TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_empty_partition_with_empty_value.db/" "append_with_empty_partition_with_empty_value/"; @@ -1878,7 +1931,7 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { const std::shared_ptr& expected) { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1921,13 +1974,13 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { } TEST_P(ScanAndReadInteTest, TestMemoryUse) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1966,7 +2019,7 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { } TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); auto test_dir = UniqueTestDirectory::Create("local"); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), @@ -2029,7 +2082,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // batch scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2039,7 +2092,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // stream scan: from snapshot 1 ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2073,7 +2126,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // stream scan: from snapshot 2 ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2101,7 +2154,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2133,7 +2186,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForParquet) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); if (file_format != "parquet") { return; } @@ -2144,7 +2197,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2152,7 +2205,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.SetReadSchema({"f10", "f8", "f4", "f13"}); + read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2182,8 +2235,58 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { + auto file_format = FileFormat(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 6); + ASSERT_EQ(result_plan->Splits().size(), 1); + + size_t data_file_count = 0; + for (const auto& split : result_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + data_file_count += data_split->data_files_.size(); + } + ASSERT_EQ(data_file_count, 1); + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("key1", arrow::int32()), arrow::field("k", arrow::utf8()), + arrow::field("key_2", arrow::int32()), + arrow::field("c", arrow::int32()), arrow::field("d", arrow::int32()), + arrow::field("a", arrow::int32()), arrow::field("key0", arrow::int32()), + arrow::field("e", arrow::int32())}), + R"([ +[0, 1, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 2, 4, null, 6, 0, null], +[0, 1, "Alice", 12, 94, null, 96, 0, null], +[0, 1, "Paul", 502, 504, 508, 506, 0, 509] +])") + .ValueOrDie()); + ASSERT_TRUE(expected); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); if (file_format != "parquet") { return; } @@ -2193,7 +2296,7 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2225,12 +2328,12 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2263,13 +2366,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2302,7 +2405,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/pk_dv_index_not_in_data_with_external"; @@ -2310,7 +2413,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { "/pk_dv_index_not_in_data_with_external.db/external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2345,7 +2448,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { } TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_bitmap.db/append_with_bitmap"; auto predicate = @@ -2390,7 +2493,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/pk_dv_index_in_data_with_external"; @@ -2398,7 +2501,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2433,12 +2536,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { } TEST_P(ScanAndReadInteTest, TestTimestampType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2479,13 +2582,13 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { TEST_P(ScanAndReadInteTest, TestCastTimestampType) { TimezoneGuard tz_guard("Asia/Shanghai"); - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2664,7 +2767,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { } TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); // Verify BucketSelectConverter: an EQUAL predicate on bucket key f2 should automatically // derive the target bucket, without explicitly calling SetBucketFilter. // From the existing test f2=0 maps to bucket 1, f2=1 maps to bucket 0. @@ -2679,7 +2782,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2716,4 +2819,102 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { + auto file_format = FileFormat(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + // CreateCountReader with empty splits should return 0 rows. + std::vector> empty_splits; + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(empty_splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, 0); +} + +TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { + auto file_format = FileFormat(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Scan latest snapshot + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + // Method 1: CreateCountReader + iterate batches + ReadContextBuilder count_context_builder(table_path); + AddReadOptionsForPrefetch(&count_context_builder); + ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN(auto count_reader, + count_table_read->CreateCountReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); + + // Method 2: CreateReader + iterate batches + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + int64_t iterate_count = read_result ? read_result->length() : 0; + + // Both methods should return the same count + ASSERT_EQ(count_result, iterate_count); +} + +TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { + auto file_format = FileFormat(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Create splits from latest snapshot. + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + // Set predicate in read context. CountReader currently does not support predicate pushdown. + auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, + Literal(static_cast(0))); + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), + "predicate pushdown is not supported"); +} + +TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported) { + auto file_format = FileFormat(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Create splits from latest snapshot. + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + auto* key_value_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(key_value_table_read != nullptr); + key_value_table_read->ForceKeepDelete(true); + + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), + "force_keep_delete is not supported"); +} + } // namespace paimon::test diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 2989cf2e..fdc38ea9 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -55,11 +56,27 @@ #include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { -class ScanInteTest : public testing::Test { +enum class ManifestCacheMode { NoCache, Cache }; + +class ScanInteTest : public testing::TestWithParam { public: + Result> FinishScanContext(ScanContextBuilder& builder) { + if (GetParam() == ManifestCacheMode::Cache) { + if (!cache_) { + cache_ = std::make_shared(std::map{ + {CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); + } + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(cache_); + } + return builder.Finish(); + } + std::vector> CollectDataSplits( const std::shared_ptr& plan) const { std::vector> result_data_splits; @@ -109,6 +126,8 @@ class ScanInteTest : public testing::Test { } private: + std::shared_ptr cache_; + std::shared_ptr pool_ = GetDefaultPool(); std::shared_ptr arrow_data_type_ = @@ -244,11 +263,143 @@ class ScanInteTest : public testing::Test { /*write_cols=*/std::nullopt); }; -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1) { +TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { + auto manifest_cache = + std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024); + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + + auto run_scan = [&]() -> Result> { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + return table_scan->CreatePlan(); + }; + + ASSERT_OK_AND_ASSIGN(auto first_plan, run_scan()); + ASSERT_TRUE(first_plan->SnapshotId()); + ASSERT_EQ(1, first_plan->SnapshotId().value()); + ASSERT_FALSE(first_plan->Splits().empty()); + ASSERT_GT(manifest_cache->SupplierCallCount(), 0); + int64_t supplier_calls_after_first_scan = manifest_cache->SupplierCallCount(); + + ASSERT_OK_AND_ASSIGN(auto second_plan, run_scan()); + ASSERT_EQ(first_plan->SnapshotId(), second_plan->SnapshotId()); + ASSERT_EQ(first_plan->Splits().size(), second_plan->Splits().size()); + ASSERT_GT(manifest_cache->GetCount(), supplier_calls_after_first_scan); + ASSERT_EQ(supplier_calls_after_first_scan, manifest_cache->SupplierCallCount()); +} + +Result>> RunBucketSnapshotScan( + const std::string& table_path, int64_t snapshot_id, int32_t bucket, + const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) + .SetBucketFilter(bucket); + if (max_snapshot_live_manifest_versions > 0) { + context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, + std::to_string(max_snapshot_live_manifest_versions)); + } + if (cache) { + context_builder.WithCache(cache); + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); + + std::vector> data_splits; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("expected DataSplitImpl from bucket snapshot scan"); + } + data_splits.push_back(data_split); + } + return data_splits; +} + +void AssertDataSplitsEqual(const std::vector>& expected, + const std::vector>& actual) { + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + ASSERT_EQ(*actual[i], *expected[i]) << actual[i]->ToString() << std::endl + << expected[i]->ToString(); + } +} + +std::shared_ptr CreateSnapshotLiveManifestTestCache() { + return std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); +} + +TEST(ScanInteManifestCacheTest, TestRepeatedBucketSnapshotScanReusesSnapshotLiveManifestCache) { + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + auto cache = CreateSnapshotLiveManifestTestCache(); + + ASSERT_OK_AND_ASSIGN(auto expected, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto first, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected, first); + ASSERT_GT(cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST), 0); + int64_t get_count_after_first_scan = cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + int64_t supplier_calls_after_first_scan = + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + + ASSERT_OK_AND_ASSIGN(auto second, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected, second); + ASSERT_EQ(get_count_after_first_scan + 1, cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); + ASSERT_EQ(supplier_calls_after_first_scan, + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); +} + +TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPerBucketValue) { + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + auto cache = CreateSnapshotLiveManifestTestCache(); + + ASSERT_OK_AND_ASSIGN(auto expected_snapshot3, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto cached_snapshot3, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot3, cached_snapshot3); + + ASSERT_OK_AND_ASSIGN(auto expected_snapshot5, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, /*bucket=*/0, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto cached_snapshot5, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot5, cached_snapshot5); + int64_t get_count_after_snapshot5 = cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + int64_t supplier_calls_after_snapshot5 = + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + + ASSERT_OK_AND_ASSIGN(auto cached_snapshot3_again, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot3, cached_snapshot3_again); + ASSERT_EQ(get_count_after_snapshot5 + 1, cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); + ASSERT_EQ(supplier_calls_after_snapshot5, + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); +} + +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -300,11 +451,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -355,36 +506,106 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanInvalidSnapshot) { +TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { + std::string data_dir = paimon::test::GetDataDir(); + if (!std::filesystem::exists(data_dir + "orc/append_09.db/append_09/schema/schema-0")) { + data_dir = "../" + data_dir; + } + std::string table_path = data_dir + "orc/append_09.db/append_09"; + + auto check_result = [&](const std::optional& specific_table_schema) { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + if (specific_table_schema) { + context_builder.SetTableSchema(specific_table_schema.value()); + } + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ASSERT_EQ(result_plan->SnapshotId().value(), 3); + + auto result_data_splits = CollectDataSplits(result_plan); + DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({10}, pool_.get()), + /*bucket=*/0, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=10/bucket-0", + {meta_snapshot1_partition10_bucket0_}); + auto expected_data_split1 = + std::dynamic_pointer_cast(builder1.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + DataSplitImpl::Builder builder2( + BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/1, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=10/bucket-1", + {meta_snapshot1_partition10_bucket1_, meta_snapshot2_partition10_bucket1_, + meta_snapshot3_partition10_bucket1_}); + auto expected_data_split2 = + std::dynamic_pointer_cast(builder2.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + DataSplitImpl::Builder builder3( + BinaryRowGenerator::GenerateRow({20}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=20/bucket-0", + {meta_snapshot1_partition20_bucket0_, meta_snapshot2_partition20_bucket0_}); + auto expected_data_split3 = + std::dynamic_pointer_cast(builder3.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + std::vector> expected_data_splits = { + expected_data_split1, expected_data_split2, expected_data_split3}; + CheckResult(expected_data_splits, result_data_splits); + }; + + check_result(std::nullopt); + + auto fs = std::make_shared(); + std::string schema_str; + ASSERT_OK(fs->ReadFile(table_path + "/schema/schema-0", &schema_str)); + check_result(std::optional(schema_str)); +} + +TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); } -TEST_F(ScanInteTest, TestBatchScanMultipleTimes) { +TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -447,11 +668,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -493,11 +714,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -535,12 +756,12 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1_1( @@ -648,12 +869,12 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1( @@ -702,11 +923,11 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { +TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -757,27 +978,27 @@ TEST_F(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) .AddOption(Options::SCAN_SNAPSHOT_ID, "2") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder2_1( @@ -842,14 +1063,14 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) .AddOption(Options::SCAN_SNAPSHOT_ID, "2") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_TRUE(scan_context); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); @@ -927,13 +1148,13 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithInvalidOptions) { +TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), @@ -942,13 +1163,13 @@ TEST_F(ScanInteTest, TestScanAppendWithInvalidOptions) { { ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::string val("Bob"); @@ -958,7 +1179,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -982,7 +1203,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { +TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; @@ -1006,7 +1227,7 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { context_builder.SetPredicate(predicate) .AddOption(Options::SCAN_SNAPSHOT_ID, "1") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1_2( @@ -1058,14 +1279,14 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::map partition_keys; partition_keys["f1"] = "10"; ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1104,19 +1325,19 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "field invalid_partition_key does not exist in partition keys"); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; // set predicate filter, f1 = 20 @@ -1130,7 +1351,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF context_builder.SetPredicate(predicate) .SetPartitionFilter({partition_keys}) .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1169,7 +1390,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { std::string table_path = paimon::test::GetDataDir() + "orc/multi_partition_append_table.db/multi_partition_append_table"; @@ -1180,7 +1401,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1221,7 +1442,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { } // test complex type ts & decimal -TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) { +TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_complex_data.db/append_complex_data"; // set predicate filter @@ -1236,7 +1457,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); @@ -1282,7 +1503,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) } // test complex type date & binary -TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) { +TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_complex_data.db/append_complex_data"; // set predicate filter @@ -1297,7 +1518,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1347,14 +1568,14 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; ScanContextBuilder context_builder(table_path); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1441,7 +1662,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; ScanContextBuilder context_builder(table_path); @@ -1452,7 +1673,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { auto predicate = PredicateBuilder::And({greater_than, equal}).value(); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1486,40 +1707,40 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { +TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanPKWithInvalidOptions) { +TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } -TEST_F(ScanInteTest, TestReadWithNoSnapshot) { +TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; @@ -1546,7 +1767,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1584,7 +1805,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; ScanContextBuilder context_builder(table_path); @@ -1596,7 +1817,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1659,7 +1880,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; @@ -1672,7 +1893,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1707,7 +1928,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; ScanContextBuilder context_builder(table_path); @@ -1719,7 +1940,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1783,7 +2004,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; ScanContextBuilder context_builder(table_path); @@ -1795,13 +2016,13 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; @@ -1814,7 +2035,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1848,7 +2069,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -1858,7 +2079,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { FieldType::INT, Literal(100)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1916,7 +2137,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -1927,7 +2148,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1990,7 +2211,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -2003,7 +2224,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { FieldType::BIGINT, Literal(100l)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2066,7 +2287,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -2081,35 +2302,36 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithTag1) { +TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); } -TEST_F(ScanInteTest, TestScanInvalidTag) { +TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } -TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { +TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { std::string table_path = GetDataDir() + "orc/append_09.db/append_09"; auto fs = std::make_shared(); @@ -2120,7 +2342,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); @@ -2129,7 +2351,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); @@ -2139,7 +2361,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); @@ -2148,14 +2370,14 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } } -TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { +TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { std::string table_path = GetDataDir() + "orc/append_09.db/append_09"; auto fs = std::make_shared(); @@ -2167,7 +2389,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2179,7 +2401,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2191,7 +2413,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2205,7 +2427,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { .AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(std::numeric_limits::max())) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2213,4 +2435,18 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } } +std::string ManifestCacheModeName(const testing::TestParamInfo& info) { + switch (info.param) { + case ManifestCacheMode::NoCache: + return "NoCache"; + case ManifestCacheMode::Cache: + return "Cache"; + } + return "Unknown"; +} + +INSTANTIATE_TEST_SUITE_P(ManifestCacheMode, ScanInteTest, + testing::Values(ManifestCacheMode::NoCache, ManifestCacheMode::Cache), + ManifestCacheModeName); + } // namespace paimon::test diff --git a/test/inte/variant_table_inte_test.cpp b/test/inte/variant_table_inte_test.cpp new file mode 100644 index 00000000..3cedac21 --- /dev/null +++ b/test/inte/variant_table_inte_test.cpp @@ -0,0 +1,879 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/data/variant.h" +#include "paimon/defs.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/reader_builder.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/record_batch.h" +#include "paimon/table/source/startup_mode.h" +#include "paimon/testing/utils/io_exception_helper.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +// End-to-end tests for tables with a VARIANT column: create, write, commit, scan and read. +class VariantTableInteTest : public ::testing::Test { + public: + void SetUp() override { + dir_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + test_dir_ = dir_->Str(); + pool_ = GetDefaultPool(); + fields_ = {arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}; + schema_ = arrow::schema(fields_); + } + + void TearDown() override { + dir_.reset(); + } + + std::shared_ptr BuildArray(const std::vector& jsons, + int32_t id_offset = 0) { + auto result = + VariantTestData::BuildVariantBatch(fields_[0], fields_[1], jsons, pool_, id_offset); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return std::move(result).value(); + } + + Result> MakeBatch( + const std::shared_ptr& array) { + ::ArrowArray arrow_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &arrow_array)); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.SetPartition({}).SetBucket(0).SetRowKinds({}).Finish(); + } + + // Reads all rows back and checks the variant column renders to `expected_jsons` (nullptr + // means a null variant). The read result carries a leading `_VALUE_KIND` column. + void ReadAndCheck(TestHelper* helper, const std::vector>& splits, + const std::vector& expected_ids, + const std::vector& expected_jsons) { + ASSERT_OK_AND_ASSIGN(auto result, helper->ReadResult(splits)); + ASSERT_EQ(result->num_chunks(), 1); + auto result_struct = std::static_pointer_cast(result->chunk(0)); + ASSERT_EQ(result_struct->length(), static_cast(expected_jsons.size())); + auto struct_type = std::static_pointer_cast(result_struct->type()); + int32_t id_index = struct_type->GetFieldIndex("id"); + int32_t variant_index = struct_type->GetFieldIndex("v"); + ASSERT_GE(id_index, 0); + ASSERT_GE(variant_index, 0); + auto id_column = + std::static_pointer_cast(result_struct->field(id_index)); + auto variant_column = + std::static_pointer_cast(result_struct->field(variant_index)); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = + std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < expected_jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + ASSERT_EQ(id_column->Value(i), expected_ids[i]); + if (expected_jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(expected_jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } + } + + // Builds a variant-access projection field via the public builder. + std::shared_ptr BuildAccessField( + const std::vector, std::string>>& accesses, + const std::string& field_name) { + VariantAccessBuilder builder; + for (const auto& [type, path] : accesses) { + auto target = std::make_unique(); + EXPECT_TRUE(arrow::ExportField(arrow::Field("t", type), target.get()).ok()); + EXPECT_OK(builder.AddField(target.get(), path, /*fail_on_error=*/false)); + } + auto c_field = builder.Build(field_name); + EXPECT_TRUE(c_field.ok()) << c_field.status().ToString(); + auto imported = arrow::ImportField(c_field.value().get()); + EXPECT_TRUE(imported.ok()) << imported.status().ToString(); + return imported.ValueOrDie(); + } + + // Reads `splits` back with `read_schema` projected and returns the single result chunk. + void ReadWithSchema(TestHelper* helper, const std::vector>& splits, + const std::shared_ptr& read_schema, + std::shared_ptr* result_struct) { + auto c_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto result, helper->ReadResult(splits, std::move(c_read_schema))); + ASSERT_EQ(result->num_chunks(), 1); + *result_struct = std::static_pointer_cast(result->chunk(0)); + } + + Result> ReadDataFileSchema( + const std::string& bucket_path, const std::shared_ptr& file, + const std::map& options) const { + std::string file_path = PathUtil::JoinPath(bucket_path, file->file_name); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr unique_input_stream, + dir_->GetFileSystem()->Open(file_path)); + std::shared_ptr input_stream(std::move(unique_input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::string format, file->FileFormat()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format, + FileFormatFactory::Get(format, options)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + reader_builder->Build(input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr c_file_schema, reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); + return file_schema; + } + + std::map AdaptiveInferenceOptions(int64_t target_file_row_num, + int32_t initial_sample_rows, + int32_t adaptive_sample_rows, + double admission_ratio, + double retention_ratio) const { + return { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + {Options::WRITE_ONLY, "true"}, + {Options::TARGET_FILE_ROW_NUM, std::to_string(target_file_row_num)}, + {Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"}, + {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"}, + {Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, std::to_string(initial_sample_rows)}, + {Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW, + std::to_string(adaptive_sample_rows)}, + {Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + fmt::format("{}", admission_ratio)}, + {Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO, + fmt::format("{}", retention_ratio)}, + }; + } + + protected: + std::string test_dir_; + std::unique_ptr dir_; + std::shared_ptr pool_; + arrow::FieldVector fields_; + std::shared_ptr schema_; +}; + +TEST_F(VariantTableInteTest, TestAppendTable) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + // The document set covers deep object/array alternation, escaped and unicode strings, + // wide integers, decimals, exponent doubles and empty containers. + std::vector jsons = { + R"({"age": 35, "city": "Hangzhou"})", + nullptr, + "[1, \"two\", 3.5, null, true]", + "{\"nested\": {\"x\": [1, 2]}, \"s\": \"中文\"}", + R"({ + "user": { + "id": 9007199254740993, + "name": "张三 \"quoted\" \\ / \b\f\n\r\t", + "tags": ["a", 1, 2.5, true, null, {"deep": [[1, [2, [3, [4]]]]]}], + "address": { + "city": "Hangzhou", + "geo": {"lat": 30.274085, "lng": 120.15507, "alt": -1.5e-3}, + "history": [ + {"year": 2020, "city": "Beijing"}, + {"year": 2021, "city": "Shanghai", "note": null} + ] + }, + "balance": 12345678901234567890.123456789, + "scores": [0.1, -0.0, 1e100, -1e-100] + }, + "empty_object": {}, + "empty_array": [], + "flags": [true, false, null] + })", + R"([{"a": [{"b": {"c": [null, {"d": 1}]}}]}, [], {}, "end"])", + R"({"unicode": "😀"})", + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ReadAndCheck(helper.get(), splits, {0, 1, 2, 3, 4, 5, 6}, jsons); +} + +TEST_F(VariantTableInteTest, TestAdaptiveInferenceAcrossRollingFiles) { + std::map options = + AdaptiveInferenceOptions(/*target_file_row_num=*/10, /*initial_sample_rows=*/10, + /*adaptive_sample_rows=*/10, /*admission_ratio=*/0.4, + /*retention_ratio=*/0.2); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + std::vector json_storage; + json_storage.reserve(30); + for (int32_t id = 0; id < 10; ++id) { + json_storage.push_back(id < 5 ? fmt::format(R"({{"legacy":"value","stable":{}}})", id) + : fmt::format(R"({{"stable":{}}})", id)); + } + for (int32_t id = 10; id < 20; ++id) { + json_storage.push_back(id < 19 ? fmt::format(R"({{"emerging":true,"stable":{}}})", id) + : fmt::format(R"({{"stable":{}}})", id)); + } + for (int32_t id = 20; id < 30; ++id) { + json_storage.push_back(fmt::format(R"({{"stable":{}}})", id)); + } + std::vector jsons; + jsons.reserve(json_storage.size()); + for (const std::string& json : json_storage) { + jsons.push_back(json.c_str()); + } + + std::vector> batches; + for (int32_t file_index = 0; file_index < 3; ++file_index) { + auto begin = jsons.begin() + file_index * 10; + std::vector file_jsons(begin, begin + 10); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(BuildArray(file_jsons, /*id_offset=*/file_index * 10))); + batches.push_back(std::move(batch)); + } + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ASSERT_EQ(1, splits.size()); + auto data_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_NE(data_split, nullptr); + std::vector> files = data_split->DataFiles(); + std::sort( + files.begin(), files.end(), + [](const std::shared_ptr& left, const std::shared_ptr& right) { + return left->min_sequence_number < right->min_sequence_number; + }); + ASSERT_EQ(3, files.size()); + + std::vector> expected_variant_types; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr first_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())}))); + expected_variant_types.push_back(first_type); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr second_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()), + arrow::field("stable", arrow::int64())}))); + expected_variant_types.push_back(second_type); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr third_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("emerging", arrow::boolean()), arrow::field("stable", arrow::int64())}))); + expected_variant_types.push_back(third_type); + + for (size_t i = 0; i < files.size(); ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema, + ReadDataFileSchema(data_split->BucketPath(), files[i], options)); + std::shared_ptr variant_field = file_schema->GetFieldByName("v"); + ASSERT_NE(variant_field, nullptr); + ASSERT_TRUE(variant_field->type()->Equals(*expected_variant_types[i])) + << "file=" << files[i]->file_name << ", actual=" << variant_field->type()->ToString() + << ", expected=" << expected_variant_types[i]->ToString(); + } + + std::vector expected_ids; + expected_ids.reserve(30); + for (int32_t id = 0; id < 30; ++id) { + expected_ids.push_back(id); + } + ReadAndCheck(helper.get(), splits, expected_ids, jsons); +} + +TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithMultipleVariantFields) { + std::shared_ptr left_field = VariantTypeUtils::ToArrowField("left_payload"); + std::shared_ptr right_field = VariantTypeUtils::ToArrowField("right_payload"); + std::shared_ptr table_schema = + arrow::schema({arrow::field("id", arrow::int32()), left_field, right_field}); + std::map options = + AdaptiveInferenceOptions(/*target_file_row_num=*/2, /*initial_sample_rows=*/2, + /*adaptive_sample_rows=*/2, /*admission_ratio=*/0.4, + /*retention_ratio=*/0.2); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::vector left_jsons = { + R"({"legacy":"a","stable":1})", + R"({"legacy":"b","stable":2})", + R"({"emerging":true,"stable":3})", + R"({"emerging":false,"stable":4})", + }; + std::vector right_jsons = { + R"({"sparse":"x","stable":"a"})", + R"({"stable":"b"})", + R"({"stable":"c"})", + R"({"emerging":true,"stable":"d"})", + }; + std::vector> batches; + for (int32_t file_index = 0; file_index < 2; ++file_index) { + auto left_begin = left_jsons.begin() + file_index * 2; + std::vector file_left_jsons(left_begin, left_begin + 2); + auto right_begin = right_jsons.begin() + file_index * 2; + std::vector file_right_jsons(right_begin, right_begin + 2); + ASSERT_OK_AND_ASSIGN(std::shared_ptr left, + VariantTestData::BuildVariantBatch(table_schema->field(0), left_field, + file_left_jsons, pool_, + /*id_offset=*/file_index * 2 + 1)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr right, + VariantTestData::BuildVariantBatch(table_schema->field(0), right_field, + file_right_jsons, pool_, + /*id_offset=*/file_index * 2 + 1)); + std::shared_ptr rows = + arrow::StructArray::Make({left->field(0), left->field(1), right->field(1)}, + table_schema->fields()) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(rows)); + batches.push_back(std::move(batch)); + } + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ASSERT_EQ(1, splits.size()); + std::shared_ptr data_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_NE(data_split, nullptr); + std::vector> files = data_split->DataFiles(); + std::sort( + files.begin(), files.end(), + [](const std::shared_ptr& left, const std::shared_ptr& right) { + return left->min_sequence_number < right->min_sequence_number; + }); + ASSERT_EQ(2, files.size()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr first_left_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())}))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr second_left_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()), + arrow::field("stable", arrow::int64())}))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr right_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("sparse", arrow::utf8()), arrow::field("stable", arrow::utf8())}))); + std::vector> expected_left_types = {first_left_type, + second_left_type}; + for (size_t i = 0; i < files.size(); ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema, + ReadDataFileSchema(data_split->BucketPath(), files[i], options)); + std::shared_ptr file_left = file_schema->GetFieldByName("left_payload"); + std::shared_ptr file_right = file_schema->GetFieldByName("right_payload"); + ASSERT_NE(file_left, nullptr); + ASSERT_NE(file_right, nullptr); + ASSERT_TRUE(file_left->type()->Equals(*expected_left_types[i])) + << "file=" << files[i]->file_name << ", actual=" << file_left->type()->ToString() + << ", expected=" << expected_left_types[i]->ToString(); + ASSERT_TRUE(file_right->type()->Equals(*right_type)) + << "file=" << files[i]->file_name << ", actual=" << file_right->type()->ToString() + << ", expected=" << right_type->ToString(); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, helper->ReadResult(splits)); + std::map actual; + for (const std::shared_ptr& chunk : result->chunks()) { + auto rows = std::static_pointer_cast(chunk); + auto row_type = std::static_pointer_cast(rows->type()); + auto ids = + std::static_pointer_cast(rows->field(row_type->GetFieldIndex("id"))); + auto left = std::static_pointer_cast( + rows->field(row_type->GetFieldIndex("left_payload"))); + auto right = std::static_pointer_cast( + rows->field(row_type->GetFieldIndex("right_payload"))); + auto left_values = std::static_pointer_cast(left->field(0)); + auto left_metadata = std::static_pointer_cast(left->field(1)); + auto right_values = std::static_pointer_cast(right->field(0)); + auto right_metadata = std::static_pointer_cast(right->field(1)); + for (int64_t i = 0; i < rows->length(); ++i) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr left_variant, + GenericVariant::Create(left_values->GetView(i), left_metadata->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr right_variant, + GenericVariant::Create(right_values->GetView(i), + right_metadata->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string left_json, left_variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::string right_json, right_variant->ToJson()); + actual.emplace(ids->Value(i), left_json + "|" + right_json); + } + } + ASSERT_EQ(4, actual.size()); + for (int32_t id = 1; id <= 4; ++id) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected_left, + GenericVariant::FromJson(left_jsons[id - 1], pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected_right, + GenericVariant::FromJson(right_jsons[id - 1], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_left_json, expected_left->ToJson()); + ASSERT_OK_AND_ASSIGN(std::string expected_right_json, expected_right->ToJson()); + ASSERT_EQ(expected_left_json + "|" + expected_right_json, actual[id]); + } +} + +TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithNestedVariant) { + std::shared_ptr nested_variant = VariantTypeUtils::ToArrowField("payload"); + std::shared_ptr nested_field = arrow::field( + "nested", arrow::struct_({arrow::field("label", arrow::utf8()), nested_variant})); + std::shared_ptr table_schema = + arrow::schema({arrow::field("id", arrow::int32()), nested_field}); + std::map options = + AdaptiveInferenceOptions(/*target_file_row_num=*/2, /*initial_sample_rows=*/2, + /*adaptive_sample_rows=*/2, /*admission_ratio=*/0.4, + /*retention_ratio=*/0.2); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::vector labels = {"first", "second", "third", "fourth"}; + std::vector jsons = { + R"({"legacy":"a","stable":1})", + R"({"legacy":"b","stable":2})", + R"({"emerging":true,"stable":3})", + R"({"emerging":false,"stable":4})", + }; + std::vector> batches; + for (int32_t file_index = 0; file_index < 2; ++file_index) { + auto json_begin = jsons.begin() + file_index * 2; + std::vector file_jsons(json_begin, json_begin + 2); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variants, + VariantTestData::BuildVariantBatch(table_schema->field(0), nested_variant, file_jsons, + pool_, /*id_offset=*/file_index * 2 + 1)); + arrow::StringBuilder label_builder; + for (int32_t i = file_index * 2; i < file_index * 2 + 2; ++i) { + arrow::Status status = label_builder.Append(labels[i]); + ASSERT_TRUE(status.ok()) << status.ToString(); + } + std::shared_ptr label_array = label_builder.Finish().ValueOrDie(); + std::shared_ptr nested = + arrow::StructArray::Make({label_array, variants->field(1)}, + nested_field->type()->fields()) + .ValueOrDie(); + std::shared_ptr rows = + arrow::StructArray::Make({variants->field(0), nested}, table_schema->fields()) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(rows)); + batches.push_back(std::move(batch)); + } + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ASSERT_EQ(1, splits.size()); + std::shared_ptr data_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_NE(data_split, nullptr); + std::vector> files = data_split->DataFiles(); + std::sort( + files.begin(), files.end(), + [](const std::shared_ptr& left, const std::shared_ptr& right) { + return left->min_sequence_number < right->min_sequence_number; + }); + ASSERT_EQ(2, files.size()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr first_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())}))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr second_type, + VariantShreddingUtils::VariantShreddingSchema(arrow::struct_( + {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()), + arrow::field("stable", arrow::int64())}))); + std::vector> expected_types = {first_type, second_type}; + for (size_t i = 0; i < files.size(); ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema, + ReadDataFileSchema(data_split->BucketPath(), files[i], options)); + std::shared_ptr file_nested = file_schema->GetFieldByName("nested"); + ASSERT_NE(file_nested, nullptr); + auto file_nested_type = std::static_pointer_cast(file_nested->type()); + std::shared_ptr file_variant = file_nested_type->GetFieldByName("payload"); + ASSERT_NE(file_variant, nullptr); + ASSERT_TRUE(file_variant->type()->Equals(*expected_types[i])) + << "file=" << files[i]->file_name << ", actual=" << file_variant->type()->ToString() + << ", expected=" << expected_types[i]->ToString(); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, helper->ReadResult(splits)); + std::map actual; + for (const std::shared_ptr& chunk : result->chunks()) { + auto rows = std::static_pointer_cast(chunk); + auto row_type = std::static_pointer_cast(rows->type()); + auto ids = + std::static_pointer_cast(rows->field(row_type->GetFieldIndex("id"))); + auto nested = std::static_pointer_cast( + rows->field(row_type->GetFieldIndex("nested"))); + auto nested_type = std::static_pointer_cast(nested->type()); + auto label_column = std::static_pointer_cast( + nested->field(nested_type->GetFieldIndex("label"))); + auto variant = std::static_pointer_cast( + nested->field(nested_type->GetFieldIndex("payload"))); + auto value_column = std::static_pointer_cast(variant->field(0)); + auto metadata_column = std::static_pointer_cast(variant->field(1)); + for (int64_t i = 0; i < rows->length(); ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr value, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, value->ToJson()); + actual.emplace(ids->Value(i), label_column->GetString(i) + "|" + json); + } + } + ASSERT_EQ(4, actual.size()); + for (int32_t id = 1; id <= 4; ++id) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[id - 1], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(std::string(labels[id - 1]) + "|" + expected_json, actual[id]); + } +} + +TEST_F(VariantTableInteTest, TestPrimaryKeyTable) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_1, + MakeBatch(BuildArray({"{\"a\": 1}", "{\"b\": 2}", nullptr}, /*id_offset=*/0))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + MakeBatch(BuildArray({"{\"b\": \"updated\"}", "[42]"}, /*id_offset=*/1))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + // The second batch overwrites ids 1 and 2, so the merged view holds three rows. + ReadAndCheck(helper.get(), splits, {0, 1, 2}, {"{\"a\": 1}", R"({"b": "updated"})", "[42]"}); +} + +TEST_F(VariantTableInteTest, TestVariantAccessRead) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = + BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}, "v"); + auto read_schema = arrow::schema({fields_[0], access_field}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + ASSERT_EQ(result_struct->length(), 3); + auto struct_type = std::static_pointer_cast(result_struct->type()); + auto v_column = std::static_pointer_cast( + result_struct->field(struct_type->GetFieldIndex("v"))); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); +} + +// The two tests below read a variant nested inside a ROW and inside an ARRAY column as a +// variant-access projection. Unlike the format-level tests they go through the whole table read +// path, where the read schema is resolved against the table schema before the read plans see it. +TEST_F(VariantTableInteTest, TestNestedRowVariantAccessRead) { + // Table: [id, s: ROW] + auto struct_field = arrow::field("s", arrow::struct_({VariantTypeUtils::ToArrowField("nv"), + arrow::field("t", arrow::utf8())})); + auto table_schema = arrow::schema({fields_[0], struct_field}); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + auto variant_batch = BuildArray(jsons); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < jsons.size(); ++i) { + ASSERT_TRUE(sibling_builder.Append("t" + std::to_string(i)).ok()); + } + std::shared_ptr sibling; + ASSERT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto struct_data = arrow::ArrayData::Make( + struct_field->type(), static_cast(jsons.size()), {nullptr}, + {variant_batch->field(1)->data(), sibling->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_({fields_[0], struct_field}), static_cast(jsons.size()), {nullptr}, + {variant_batch->field(0)->data(), struct_data}, /*null_count=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(std::make_shared(batch_data))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = + BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}, "nv"); + auto read_schema = arrow::schema( + {fields_[0], + struct_field->WithType(arrow::struct_({access_field, struct_field->type()->field(1)}))}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + + auto struct_type = std::static_pointer_cast(result_struct->type()); + auto s_column = std::static_pointer_cast( + result_struct->field(struct_type->GetFieldIndex("s"))); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + const auto& kept_sibling = static_cast(*s_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(nv.IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + for (size_t i = 0; i < jsons.size(); ++i) { + EXPECT_EQ(kept_sibling.GetString(static_cast(i)), "t" + std::to_string(i)); + } +} + +TEST_F(VariantTableInteTest, TestArrayVariantAccessRead) { + // Table: [id, arr: ARRAY] + auto list_field = arrow::field("arr", arrow::list(VariantTypeUtils::ToArrowField("element"))); + auto table_schema = arrow::schema({fields_[0], list_field}); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // Row 0 holds two elements, row 1 is empty and row 2 holds one. + std::vector flat = {R"({"x": 1, "y": 2})", R"({"x": 3})", R"({"x": 5})"}; + std::vector offsets = {0, 2, 2, 3}; + arrow::Int32Builder offset_builder; + ASSERT_TRUE(offset_builder.AppendValues(offsets).ok()); + std::shared_ptr offset_array; + ASSERT_TRUE(offset_builder.Finish(&offset_array).ok()); + auto elements = BuildArray(flat); + arrow::Int32Builder id_builder; + ASSERT_TRUE(id_builder.AppendValues({0, 1, 2}).ok()); + std::shared_ptr ids; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + auto list_data = arrow::ArrayData::Make(list_field->type(), /*length=*/3, + {nullptr, offset_array->data()->buffers[1]}, + {elements->field(1)->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make(arrow::struct_({fields_[0], list_field}), /*length=*/3, + {nullptr}, {ids->data(), list_data}, /*null_count=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(std::make_shared(batch_data))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "element"); + auto read_schema = arrow::schema({fields_[0], list_field->WithType(arrow::list(access_field))}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + + auto struct_type = std::static_pointer_cast(result_struct->type()); + const auto& list = static_cast( + *result_struct->field(struct_type->GetFieldIndex("arr"))); + ASSERT_EQ(list.length(), 3); + ASSERT_EQ(list.value_length(0), 2); + ASSERT_EQ(list.value_length(1), 0); + ASSERT_EQ(list.value_length(2), 1); + const auto& x = static_cast( + *static_cast(*list.values()).field(0)); + ASSERT_EQ(x.Value(list.value_offset(0)), 1); + ASSERT_EQ(x.Value(list.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(list.value_offset(2)), 5); +} + +TEST_F(VariantTableInteTest, TestReadWithIOException) { + // Injects an IO error at every position of the scan+read path and verifies each failure + // surfaces as a clean error status. + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + std::vector jsons = {R"({"age": 35, "city": "Hangzhou"})", nullptr, "[1, 2, 3]"}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + bool run_complete = false; + auto io_hook = IOHook::GetInstance(); + for (size_t i = 0; i < 500; i++) { + ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); + io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + Result>> splits = + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false); + CHECK_HOOK_STATUS(splits.status(), i); + Result> read_result = + helper->ReadResult(splits.value()); + CHECK_HOOK_STATUS(read_result.status(), i); + run_complete = true; + // All IO succeeded before the injected position was reached: check the data. + io_hook->Clear(); + ReadAndCheck(helper.get(), splits.value(), {0, 1, 2}, jsons); + break; + } + ASSERT_TRUE(run_complete); +} + +TEST_F(VariantTableInteTest, TestWriteWithIOException) { + // Injects an IO error at every position of the create+write+commit path (on a fresh table + // directory per attempt) and verifies each failure surfaces as a clean error status. + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + std::vector jsons = {R"({"age": 35, "city": "Hangzhou"})", nullptr}; + bool run_complete = false; + auto io_hook = IOHook::GetInstance(); + for (size_t i = 0; i < 500; i++) { + std::string table_dir = test_dir_ + fmt::format("/io_exception_{}", i); + ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); + io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + Result> helper = + TestHelper::Create(table_dir, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false); + CHECK_HOOK_STATUS(helper.status(), i); + Result> batch = MakeBatch(BuildArray(jsons)); + CHECK_HOOK_STATUS(batch.status(), i); + Result>> commit_msgs = + helper.value()->WriteAndCommit(std::move(batch).value(), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt); + CHECK_HOOK_STATUS(commit_msgs.status(), i); + run_complete = true; + // All IO succeeded before the injected position was reached: check the data. + io_hook->Clear(); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper.value()->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ReadAndCheck(helper.value().get(), splits, {0, 1}, jsons); + break; + } + ASSERT_TRUE(run_complete); +} + +TEST_F(VariantTableInteTest, TestOrcFormatRejected) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray({"{\"a\": 1}"}))); + auto result = helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt); + ASSERT_FALSE(result.ok()); +} + +} // namespace paimon::test diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 3f9fc624..a18fe9d2 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -25,23 +25,46 @@ #include #include +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" #include "arrow/type.h" +#include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/commit_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/startup_mode.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" - -namespace paimon { -class DataSplit; -class RecordBatch; -} // namespace paimon +#include "paimon/testing/utils/timezone_guard.h" +#include "paimon/write_context.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" namespace paimon::test { // This is a sdk end-to-end test demo that supports write, commit, scan, and read operations. @@ -65,6 +88,167 @@ class WriteAndReadInteTest return new_options; } + Status WriteNextSchema(const std::vector& fields, int32_t highest_field_id, + const std::map& options) const { + auto file_system = dir_->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(file_system, table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->fields_ = fields; + next_schema->highest_field_id_ = highest_field_id; + next_schema->options_ = options; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return file_system->AtomicStore(schema_path, schema_content); + } + + Status WriteNextSchemaWithRawFieldTypes(const std::string& fields_json, + int32_t highest_field_id) const { + auto file_system = dir_->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(file_system, table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->highest_field_id_ = highest_field_id; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + + rapidjson::Document schema_doc; + schema_doc.Parse(schema_content.c_str()); + rapidjson::Document fields_doc; + fields_doc.Parse(fields_json.c_str()); + if (schema_doc.HasParseError() || fields_doc.HasParseError() || + !schema_doc.HasMember("fields")) { + return Status::Invalid("failed to assemble schema json with raw field types"); + } + schema_doc["fields"].CopyFrom(fields_doc, schema_doc.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + schema_doc.Accept(writer); + + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return file_system->AtomicStore(schema_path, std::string(buffer.GetString())); + } + + Result ReadAndCheckProjectedResult(const std::map& options, + const std::vector& read_fields, + const std::shared_ptr& expected_type, + const std::string& expected_data) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto expected, arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data)); + return std::make_shared(expected)->Equals(actual); + } + + /// Read with a custom Arrow read schema (supports nested column pruning and + /// paimon.map.selected-keys metadata for shared-shredding partial key recall). + Result ReadAndCheckWithReadSchema(const std::map& options, + const std::shared_ptr& read_schema, + const std::shared_ptr& expected_type, + const std::string& expected_data) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + + auto c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto expected, arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data)); + return std::make_shared(expected)->Equals(actual); + } + + Result> InnerScan( + const std::map& options) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, + StartupMode::LatestFull().ToString()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + return table_scan->CreatePlan(); + } + + Result>>> CurrentDataFiles( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::vector>> files; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("split cannot be cast to DataSplitImpl"); + } + for (const auto& data_file : data_split->DataFiles()) { + files.emplace_back(data_split->BucketPath(), data_file); + } + } + std::sort(files.begin(), files.end(), [](const auto& left, const auto& right) { + return left.second->min_sequence_number < right.second->min_sequence_number; + }); + return files; + } + + Result> ReadDataFileSchema( + const std::string& bucket_path, const std::shared_ptr& file, + const std::map& options) const { + std::string file_path = PathUtil::JoinPath(bucket_path, file->file_name); + PAIMON_ASSIGN_OR_RAISE(auto unique_input_stream, dir_->GetFileSystem()->Open(file_path)); + std::shared_ptr input_stream(std::move(unique_input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::string format_str, file->FileFormat()); + PAIMON_ASSIGN_OR_RAISE(auto file_format, FileFormatFactory::Get(format_str, options)); + PAIMON_ASSIGN_OR_RAISE(auto reader_builder, file_format->CreateReaderBuilder(10)); + PAIMON_ASSIGN_OR_RAISE(auto reader, reader_builder->Build(input_stream)); + PAIMON_ASSIGN_OR_RAISE(auto c_file_schema, reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto file_schema, + arrow::ImportSchema(c_file_schema.get())); + return file_schema; + } + + Result ReadShreddingMeta( + const std::pair>& file, + const std::string& field_name, const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(auto file_schema, + ReadDataFileSchema(file.first, file.second, options)); + std::shared_ptr field = file_schema->GetFieldByName(field_name); + if (!field) { + return Status::Invalid( + fmt::format("field {} not found in data file schema", field_name)); + } + std::shared_ptr metadata = field->metadata(); + if (!metadata) { + return Status::Invalid( + fmt::format("field {} has no shared-shredding metadata", field_name)); + } + std::shared_ptr metadata_copy = metadata->Copy(); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata_copy)) { + return Status::Invalid( + fmt::format("field {} has no shared-shredding metadata", field_name)); + } + return MapSharedShreddingUtils::DeserializeMetadata(metadata_copy); + } + private: std::string test_dir_; std::unique_ptr dir_; @@ -181,10 +365,8 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) { } TEST_P(WriteAndReadInteTest, TestNestedType) { - // Represent a map as list(struct(key, value)) for cross-format compatibility. arrow::FieldVector fields = { - arrow::field("f1", arrow::list(arrow::struct_({arrow::field("key", arrow::int8()), - arrow::field("value", arrow::int16())}))), + arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())), arrow::field("f2", arrow::list(arrow::float32())), arrow::field("f3", arrow::struct_({arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int64())})), @@ -240,6 +422,95 @@ TEST_P(WriteAndReadInteTest, TestNestedType) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) { + auto [file_format, file_system] = GetParam(); + if (file_format == "lance" || file_format == "avro") { + return; + } + auto list_struct = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + auto map_value = arrow::struct_({arrow::field("m1", arrow::int32())}); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("items", arrow::list(arrow::field("item", list_struct))), + arrow::field("props", arrow::map(arrow::utf8(), map_value)), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + // Exercise ORC lazy decoding, which returns nested strings as dictionaries. + {"orc.read.enable-lazy-decoding", "true"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, + options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [[10, "x"], [20, "y"]], [["k1", [100]]]], + [2, [[30, "z"]], [["k2", [200]]]] + ])", + /*partition_map=*/{}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(auto schema_v0, schema_manager.ReadSchema(0)); + std::vector fields_v0 = schema_v0->Fields(); + int32_t next_id = schema_v0->HighestFieldId(); + + auto items_field = fields_v0[1].ArrowField(); + auto items_list = arrow::internal::checked_pointer_cast(items_field->type()); + auto items_struct = + arrow::internal::checked_pointer_cast(items_list->value_type()); + auto c_field = DataField::ConvertDataFieldToArrowField( + DataField(++next_id, arrow::field("c", arrow::int32()))); + auto new_items_type = arrow::list(items_list->value_field()->WithType( + arrow::struct_({items_struct->field(0), items_struct->field(1), c_field}))); + + auto props_field = fields_v0[2].ArrowField(); + auto props_map = arrow::internal::checked_pointer_cast(props_field->type()); + auto props_value = + arrow::internal::checked_pointer_cast(props_map->item_type()); + auto m2_field = DataField::ConvertDataFieldToArrowField( + DataField(++next_id, arrow::field("m2", arrow::int32()))); + auto new_props_type = arrow::map( + props_map->key_type(), + props_map->item_field()->WithType(arrow::struct_({props_value->field(0), m2_field}))); + + std::vector fields_v1 = fields_v0; + fields_v1[1] = DataField(fields_v0[1].Id(), items_field->WithType(new_items_type)); + fields_v1[2] = DataField(fields_v0[2].Id(), props_field->WithType(new_props_type)); + ASSERT_OK(WriteNextSchema(fields_v1, next_id, options)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("items", arrow::list(arrow::field( + "item", arrow::struct_({arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + arrow::field("c", arrow::int32())})))), + arrow::field("props", arrow::map(arrow::utf8(), + arrow::struct_({arrow::field("m1", arrow::int32()), + arrow::field("m2", arrow::int32())}))), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [[10, "x", null], [20, "y", null]], [["k1", [100, null]]]], + [0, 2, [[30, "z", null]], [["k2", [200, null]]]] + ])")); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestAppendExternalPath) { arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), @@ -852,11 +1123,73 @@ TEST_P(WriteAndReadInteTest, TestWriteSamePartitionTwiceWithAllBasicTypesForPk) ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + arrow::FieldVector fields = { + arrow::field("c", arrow::utf8()), + arrow::field("vc", arrow::utf8()), + arrow::field("b", arrow::binary()), + arrow::field("vb", arrow::binary()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK(WriteNextSchemaWithRawFieldTypes(R"json([ + { "id" : 0, "name" : "c", "type" : "CHAR(10)" }, + { "id" : 1, "name" : "vc", "type" : "VARCHAR(20)" }, + { "id" : 2, "name" : "b", "type" : "BINARY(10)" }, + { "id" : 3, "name" : "vb", "type" : "VARBINARY(20)" } + ])json", + /*highest_field_id=*/3)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + + std::string data = R"([ + ["alice", "hello world", "abc", "xyz123"], + [null, null, null, null] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, "alice", "hello world", "abc", "xyz123"], + [0, null, null, null, null] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); +} + std::vector> GetTestValuesForWriteAndReadInteTest() { std::vector> values = {{"parquet", "local"}}; +#if defined(PAIMON_ENABLE_NETWORK_TESTS) && defined(PAIMON_ENABLE_JINDO) + values.emplace_back("parquet", "jindo"); +#endif #ifdef PAIMON_ENABLE_ORC values.emplace_back("orc", "local"); - // values.emplace_back("parquet", "jindo"); #endif #ifdef PAIMON_ENABLE_AVRO values.emplace_back("avro", "local"); @@ -864,6 +1197,2345 @@ std::vector> GetTestValuesForWriteAndReadInt return values; } +/// End-to-end test for parquet page-level filtering with a PK table. +/// Writes data with page index enabled and small page size so multiple pages are created, +/// then reads with a PK equality predicate and verifies only matching rows are returned. +TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::utf8()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, "local"}, + // Force exactly one row per parquet page. Parquet's writer checks the page + // byte threshold only after every `write_batch_size` values, so the default + // batch=1024 packs all rows into a single page regardless of page.size. + // write.batch-size=1 + page.size=1 + no dictionary together guarantee that + // every value triggers a page flush, giving ColumnIndexFilter pages whose + // min == max == that row's value. With predicate f0="Alice", exactly one + // page survives page pruning, so the reader emits exactly one row -- and + // that result is attributable purely to page filtering (no row-level + // filter is enabled below). + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{"f1"}, + /*primary_keys=*/{"f0", "f1"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + int64_t commit_identifier = 0; + + // Write data: 12 rows across 2 partitions + std::string data_p1 = R"([ + ["Alice", "p1", 10, 1.1], + ["Bob", "p1", 20, 2.2], + ["Cathy", "p1", 30, 3.3], + ["David", "p1", 40, 4.4], + ["Emily", "p1", 50, 5.5], + ["Frank", "p1", 60, 6.6] + ])"; + std::string data_p2 = R"([ + ["Grace", "p2", 70, 7.7], + ["Helen", "p2", 80, 8.8], + ["Ivan", "p2", 90, 9.9], + ["Jack", "p2", 100, 10.1], + ["Kate", "p2", 110, 11.2], + ["Lucy", "p2", 120, 12.3] + ])"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p1, + /*partition_map=*/{{"f1", "p1"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p2, + /*partition_map=*/{{"f1", "p2"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_p1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_p2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Scan with PK predicate: f0 = "Alice" + std::string literal_str = "Alice"; + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 2); + ASSERT_FALSE(result_plan->Splits().empty()); + + // Read with predicate but WITHOUT EnablePredicateFilter -- so any narrowing + // of the result is attributable to split/file/RG/page pruning, not to a + // post-read row-level filter. This is what makes the exact assertion below + // meaningful as a check that page-index filtering is wired and working. + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Expected: p2 file is pruned by file-level min/max key stats (f0 range + // [Grace, Lucy] doesn't overlap "Alice"). Inside p1's file, write.batch-size=1 + // + page.size=1 produces one row per page, so page-index filter keeps only + // the page whose min == max == "Alice" -- one row. + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ +[0, "Alice", "p1", 10, 1.1] +])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + +/// End-to-end test for parquet page-level filtering on an append-only table. +/// Append-only tables read parquet files directly without PK merge, so the result +/// reflects exactly what survives row-group and page-index pruning. +TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::utf8()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}, + // Force exactly one row per parquet page (see the PK variant for why these + // three options together are required). With one row per page, + // ColumnIndexFilter keeps only the page whose min == max == "Alice", and + // without row-level filter the reader output is precisely that one row. + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + int64_t commit_identifier = 0; + + // Write data: 12 rows across 2 partitions. + std::string data_p1 = R"([ + ["Alice", "p1", 10, 1.1], + ["Bob", "p1", 20, 2.2], + ["Cathy", "p1", 30, 3.3], + ["David", "p1", 40, 4.4], + ["Emily", "p1", 50, 5.5], + ["Frank", "p1", 60, 6.6] + ])"; + std::string data_p2 = R"([ + ["Grace", "p2", 70, 7.7], + ["Helen", "p2", 80, 8.8], + ["Ivan", "p2", 90, 9.9], + ["Jack", "p2", 100, 10.1], + ["Kate", "p2", 110, 11.2], + ["Lucy", "p2", 120, 12.3] + ])"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p1, + /*partition_map=*/{{"f1", "p1"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p2, + /*partition_map=*/{{"f1", "p2"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_p1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_p2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Predicate: f0 = "Alice" + std::string literal_str = "Alice"; + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 2); + ASSERT_FALSE(result_plan->Splits().empty()); + + // Read with predicate but WITHOUT EnablePredicateFilter, so the narrowing + // observed below is attributable to page-index filtering rather than a + // post-read row-level filter. + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Partition p2's row groups don't overlap "Alice" (min/max f0 in [Grace, Lucy]), + // so the whole file is skipped. Within p1, page-index pruning narrows down to the + // page containing "Alice". With no PK merge, the result is exactly that one row. + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ +[0, "Alice", "p1", 10, 1.1] +])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + +TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + + std::string data = R"([ + ["banana", 2], + ["dog", 1], + ["lucy", 14], + ["mouse", 100] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ + [0, "banana", 2], + [0, "dog", 1], + [0, "lucy", 14], + [0, "mouse", 100] + ])") + .ValueOrDie()); + + auto cache = + std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); + auto read_once = [&]() -> Result { + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); + if (result_plan->SnapshotId() != std::optional(1)) { + return Status::Invalid("unexpected snapshot id"); + } + if (result_plan->Splits().empty()) { + return Status::Invalid("no splits found"); + } + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.WithCache(cache); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (!read_result) { + return Status::Invalid("read result is null"); + } + return expected->Equals(read_result); + }; + + ASSERT_OK_AND_ASSIGN(bool first_success, read_once()); + ASSERT_TRUE(first_success); + ASSERT_EQ(1, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); + + ASSERT_OK_AND_ASSIGN(bool second_success, read_once()); + ASSERT_TRUE(second_success); + ASSERT_EQ(2, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); +} + +TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]], + [3, null], + [4, [["d", 60], ["a", null], ["c", 70]]] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, 1, [["a", 10], ["b", 20]]], + [0, 2, [["a", 40], ["b", 50], ["c", 30]]], + [0, 3, null], + [0, 4, [["a", null], ["c", 70], ["d", 60]]] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("plain_metrics", map_type), + arrow::field("sequential_metrics", map_type), + arrow::field("lru_metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.plain_metrics.map.storage-layout", "shared-shredding"}, + {"fields.plain_metrics.map.shared-shredding.max-columns", "3"}, + {"fields.plain_metrics.map.shared-shredding.column-placement-policy", "plain"}, + {"fields.sequential_metrics.map.storage-layout", "shared-shredding"}, + {"fields.sequential_metrics.map.shared-shredding.max-columns", "3"}, + {"fields.sequential_metrics.map.shared-shredding.column-placement-policy", "sequential"}, + {"fields.lru_metrics.map.storage-layout", "shared-shredding"}, + {"fields.lru_metrics.map.shared-shredding.max-columns", "3"}, + {"fields.lru_metrics.map.shared-shredding.column-placement-policy", "lru"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20], ["c", 30]], [["a", 10], ["c", 30], ["d", 60]], [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 40], ["b", 50]], [["a", 40], ["c", 50]], [["a", 40], ["b", 50]]], + [3, [["d", 60]], [["b", 60]], [["d", 60]]], + [4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(1, files.size()); + auto check_meta = [&](const std::string& field_name, + const std::string& overflow_name) -> Status { + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta, + ReadShreddingMeta(files[0], field_name, options)); + if (meta.name_to_id.size() != 4 || meta.num_columns != 3 || meta.max_row_width != 4 || + meta.overflow_field_set.size() != 1) { + return Status::Invalid("unexpected shared-shredding metadata for ", field_name); + } + auto overflow_id = meta.name_to_id.find(overflow_name); + if (overflow_id == meta.name_to_id.end() || + meta.overflow_field_set.count(overflow_id->second) != 1) { + return Status::Invalid("unexpected overflow field for ", field_name); + } + return Status::OK(); + }; + ASSERT_OK(check_meta("plain_metrics", "d")); + ASSERT_OK(check_meta("sequential_metrics", "b")); + ASSERT_OK(check_meta("lru_metrics", "c")); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["b", 20], ["c", 30]], [["a", 10], ["c", 30], ["d", 60]], [["a", 10], ["b", 20], ["c", 30]]], + [0, 2, [["a", 40], ["b", 50]], [["a", 40], ["c", 50]], [["a", 40], ["b", 50]]], + [0, 3, [["d", 60]], [["b", 60]], [["d", 60]]], + [0, 4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("dt", arrow::utf8()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "5"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN( + auto p1_bucket0_first, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, "p1", [["a", 1]]]])", + /*partition_map=*/{{"dt", "p1"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p1_bucket0_first), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto p1_bucket0_second, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[2, "p1", [["b", 2]]]])", + /*partition_map=*/{{"dt", "p1"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p1_bucket0_second), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto p2_bucket1_first, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[3, "p2", [["x", 10], ["y", 20], ["z", 30], ["w", 40]]]])", + /*partition_map=*/{{"dt", "p2"}}, /*bucket=*/1, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p2_bucket1_first), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_GE(data_splits.size(), 2); + std::string expected_data = R"([ + [0, 1, "p1", [["a", 1]]], + [0, 2, "p1", [["b", 2]]], + [0, 3, "p2", [["w", 40], ["x", 10], ["y", 20], ["z", 30]]] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(3, files.size()); + std::vector>> p1_bucket0_files; + std::vector>> p2_bucket1_files; + for (const auto& file : files) { + if (file.first.find("dt=p1/bucket-0") != std::string::npos) { + p1_bucket0_files.push_back(file); + } else if (file.first.find("dt=p2/bucket-1") != std::string::npos) { + p2_bucket1_files.push_back(file); + } + } + ASSERT_EQ(2, p1_bucket0_files.size()); + ASSERT_EQ(1, p2_bucket1_files.size()); + + ASSERT_OK_AND_ASSIGN(auto p1_first_meta, + ReadShreddingMeta(p1_bucket0_files[0], "tags", options)); + ASSERT_EQ(5, p1_first_meta.num_columns); + ASSERT_EQ(1, p1_first_meta.max_row_width); + + ASSERT_OK_AND_ASSIGN(auto p1_second_meta, + ReadShreddingMeta(p1_bucket0_files[1], "tags", options)); + ASSERT_EQ(5, p1_second_meta.num_columns); + ASSERT_EQ(1, p1_second_meta.max_row_width); + + ASSERT_OK_AND_ASSIGN(auto p2_first_meta, + ReadShreddingMeta(p2_bucket1_files[0], "tags", options)); + ASSERT_EQ(5, p2_first_meta.num_columns); + ASSERT_EQ(4, p2_first_meta.max_row_width); +} + +TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + (void)helper; + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + auto write_one_row = [&](const std::string& data) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + return file_store_write->Write(std::move(batch)); + }; + + ASSERT_OK(write_one_row(R"([[1, [["a", 10], ["b", 20]]]])")); + ASSERT_OK(write_one_row(R"([[12, [["c", 31], ["d", 41]]]])")); + ASSERT_OK(write_one_row(R"([[21, [["e", 50], ["f", 60]]]])")); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + file_store_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/0)); + ASSERT_OK(file_store_write->Close()); + + CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); + + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(10)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options) + .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 12, [["c", 31], ["d", 41]]], + [0, 21, [["e", 50], ["f", 60]]] + ])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnCount) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "8"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN( + auto batch_v0, TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, [["a", 11]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto batch_v1, TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[2, [["b", 22]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(2, files.size()); + ASSERT_OK_AND_ASSIGN(auto first_meta, ReadShreddingMeta(files[0], "metrics", options)); + ASSERT_EQ(8, first_meta.num_columns); + ASSERT_EQ(1, first_meta.max_row_width); + ASSERT_OK_AND_ASSIGN(auto second_meta, ReadShreddingMeta(files[1], "metrics", options)); + ASSERT_EQ(8, second_meta.num_columns); + ASSERT_EQ(1, second_meta.max_row_width); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11]]], + [0, 2, [["b", 22]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::TARGET_FILE_ROW_NUM, "2"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "8"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::vector> batches; + ASSERT_OK_AND_ASSIGN( + auto first_batch, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 11], ["b", 12]]], [2, [["c", 21], ["d", 22], ["e", 23]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + batches.push_back(std::move(first_batch)); + ASSERT_OK_AND_ASSIGN( + auto second_batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[3, [["f", 31]]], [4, [["g", 41], ["h", 42]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + batches.push_back(std::move(second_batch)); + ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(2, files.size()); + ASSERT_OK_AND_ASSIGN(auto first_meta, ReadShreddingMeta(files[0], "metrics", options)); + ASSERT_EQ(8, first_meta.num_columns); + ASSERT_EQ(3, first_meta.max_row_width); + ASSERT_OK_AND_ASSIGN(auto second_meta, ReadShreddingMeta(files[1], "metrics", options)); + ASSERT_EQ(3, second_meta.num_columns); + ASSERT_EQ(2, second_meta.max_row_width); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11], ["b", 12]]], + [0, 2, [["c", 21], ["d", 22], ["e", 23]]], + [0, 3, [["f", 31]]], + [0, 4, [["g", 41], ["h", 42]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColumnsWithoutMetadata) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + arrow::field("labels", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"fields.labels.map.storage-layout", "shared-shredding"}, + {"fields.labels.map.shared-shredding.max-columns", "4"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 11], ["b", 12]], [["x", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.metrics.map.storage-layout"] = "shared-shredding"; + options_v1["fields.metrics.map.shared-shredding.max-columns"] = "3"; + options_v1["fields.labels.map.storage-layout"] = "default"; + options_v1.erase("fields.labels.map.shared-shredding.max-columns"); + ASSERT_OK( + WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])}, + /*highest_field_id=*/2, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), + options_v1, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [2, [["c", 31]], [["y", 41], ["z", 42]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options_v1)); + ASSERT_EQ(2, files.size()); + ASSERT_OK_AND_ASSIGN(auto metrics_meta, ReadShreddingMeta(files[1], "metrics", options_v1)); + ASSERT_EQ(3, metrics_meta.num_columns); + ASSERT_EQ(1, metrics_meta.max_row_width); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + arrow::field("labels", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11], ["b", 12]], [["x", 21]]], + [0, 2, [["c", 31]], [["y", 41], ["z", 42]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields_v0 = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields_v0), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([ + [1, [["a", 11], ["b", 12]]], + [2, [["c", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_v1 = { + arrow::field("id", arrow::int32()), + arrow::field("renamed_metrics", map_type), + }; + std::map options_v1 = options_v0; + options_v1.erase("fields.metrics.map.storage-layout"); + options_v1.erase("fields.metrics.map.shared-shredding.max-columns"); + options_v1["fields.renamed_metrics.map.storage-layout"] = "shared-shredding"; + options_v1["fields.renamed_metrics.map.shared-shredding.max-columns"] = "2"; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v1[0]), DataField(1, fields_v1[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), + options_v1, /*is_streaming_mode=*/false)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("renamed_metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11], ["b", 12]]], + [0, 2, [["c", 21]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields_v0 = { + arrow::field("f0", map_type), + arrow::field("f1", map_type), + arrow::field("k1", arrow::utf8()), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.f0.map.storage-layout", "shared-shredding"}, + {"fields.f0.map.shared-shredding.max-columns", "1"}, + {"fields.f1.map.storage-layout", "shared-shredding"}, + {"fields.f1.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields_v0), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([ + [[["a", 10], ["z", 11]], [["b", 20]], "old-1"], + [[["a", 12]], [["b", 21], ["y", 22]], "old-2"] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_v1 = { + arrow::field("f0", map_type), + arrow::field("f1", map_type), + arrow::field("k2", arrow::utf8()), + arrow::field("f2", map_type), + }; + std::map options_v1 = options_v0; + options_v1["fields.f2.map.storage-layout"] = "shared-shredding"; + options_v1["fields.f2.map.shared-shredding.max-columns"] = "1"; + std::vector data_fields_v1 = { + DataField(0, fields_v1[0]), + DataField(1, fields_v1[1]), + DataField(2, fields_v1[2]), + DataField(3, fields_v1[3]), + }; + ASSERT_OK(WriteNextSchema(data_fields_v1, /*highest_field_id=*/3, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), + R"([ + [[["a", 30], ["z", 31]], [["b", 40]], "new-1", [["c", 50], ["x", 51]]], + [[["a", 32]], [["b", 41]], "new-2", [["c", 52]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", map_type), + arrow::field("f2", map_type), + arrow::field("k2", arrow::utf8()), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckProjectedResult(options_v1, {"f0", "f2", "k2"}, expected_type, + R"([ + [0, [["a", 10], ["z", 11]], null, "old-1"], + [0, [["a", 12]], null, "old-2"], + [0, [["a", 30], ["z", 31]], [["c", 50], ["x", 51]], "new-1"], + [0, [["a", 32]], [["c", 52]], "new-2"] + ])")); + ASSERT_TRUE(success); +} + +// Verify storage-layout evolution: default->shared-shredding. +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, null]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[3, [["a", 30], ["z", 31]]], [4, [["a", 40]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["z", 11]]], + [0, 2, null], + [0, 3, [["a", 30], ["z", 31]]], + [0, 4, [["a", 40]]] + ])")); + ASSERT_TRUE(success); +} + +// Verify storage-layout evolution: shared-shredding->default. +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, [["a", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[3, [["a", 30], ["z", 31]]], [4, null]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["z", 11]]], + [0, 2, [["a", 20]]], + [0, 3, [["a", 30], ["z", 31]]], + [0, 4, null] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultCompaction) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + int64_t commit_identifier = 0; + + ASSERT_OK_AND_ASSIGN(auto batch_v0_file1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 11]]], + [2, [["c", 20]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0_file1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0_file2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["d", 30], ["e", 31]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0_file2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(table_path, options_v1, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto batch_v1_file3, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [4, [["a", 40], ["f", 41]]], + [5, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1_file3), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + WriteContextBuilder write_context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN( + auto write_context, + write_context_builder.SetOptions(options_v1).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN(auto compact_messages, file_store_write->PrepareCommit( + /*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(compact_messages.empty()); + + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(auto commit_context, + commit_context_builder.SetOptions(options_v1).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(file_store_commit->Commit(compact_messages, commit_identifier)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(1, splits.size()); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["b", 11]]], + [0, 2, [["c", 20]]], + [0, 3, [["d", 30], ["e", 31]]], + [0, 4, [["a", 40], ["f", 41]]], + [0, 5, null] + ])")); + ASSERT_TRUE(success); +} + +// Nested map values through both selected physical columns and overflow. +TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto value_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int32()), + }); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), value_type)), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", ["alice", 10]], ["z", ["zoe", 11]]]], + [2, [["a", ["amy", null]]]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", ["alice", 10]], ["z", ["zoe", 11]]]], + [0, 2, [["a", ["amy", null]]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto value_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("scores", arrow::list(arrow::int32())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto map_type = arrow::map(arrow::utf8(), value_type); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [ + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]], + ["z", ["zeta", [9], [["iz", 90]]]] + ]], + [2, [ + ["a", ["amy", null, [["ia", 30]]]], + ["b", ["beta", [], []]] + ]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool full_success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [ + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]], + ["z", ["zeta", [9], [["iz", 90]]]] + ]], + [0, 2, [ + ["a", ["amy", null, [["ia", 30]]]], + ["b", ["beta", [], []]] + ]], + [0, 3, null] + ])")); + ASSERT_TRUE(full_success); + + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"z,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool selected_success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [ + ["z", ["zeta", [9], [["iz", 90]]]], + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]] + ]], + [0, 2, [ + ["a", ["amy", null, [["ia", 30]]]] + ]], + [0, 3, null] + ])")); + ASSERT_TRUE(selected_success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueTypes) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + TimezoneGuard timezone_guard("Asia/Shanghai"); + // Nested TIMESTAMP_LTZ needs a timezone projection inside MAP. The Parquet reader currently + // rejects that projection, so only ORC exercises LTZ here; Parquet still covers both timestamp + // precisions as timezone-free timestamps. + std::string timezone = + file_format == "orc" ? DateTimeUtils::GetLocalTimezoneName() : std::string(); + auto value_type = arrow::struct_({ + arrow::field("bool", arrow::boolean()), + arrow::field("tiny", arrow::int8()), + arrow::field("small", arrow::int16()), + arrow::field("i", arrow::int32()), + arrow::field("big", arrow::int64()), + arrow::field("f", arrow::float32()), + arrow::field("d", arrow::float64()), + arrow::field("s", arrow::utf8()), + arrow::field("bin", arrow::binary()), + arrow::field("compact_decimal", arrow::decimal128(10, 2)), + arrow::field("large_decimal", arrow::decimal128(23, 5)), + arrow::field("date", arrow::date32()), + arrow::field("ts3", arrow::timestamp(arrow::TimeUnit::MILLI)), + arrow::field("ts9", arrow::timestamp(arrow::TimeUnit::NANO)), + arrow::field("ts_ltz3", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)), + arrow::field("ts_ltz6", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)), + arrow::field("ints", arrow::list(arrow::int32())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("nested", arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int32()), + })), + }); + auto map_type = arrow::map(arrow::utf8(), value_type); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"orc.timestamp-ltz.legacy.type", "false"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [ + ["fixed-a", [ + true, 1, 2, 3, 4, 5.5, 6.25, "str", "bin", + "12345678.90", "123456789012345678.12345", 19500, + "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456789", + "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456", + [7, null, 8], [["m1", 10], ["m2", null]], ["nested", 99] + ]], + ["fixed-b", [ + null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null + ]], + ["overflow", [ + false, 9, 10, 11, 12, 13.5, 14.25, "overflow", "raw", + "-1.23", "-123456789012345678.12345", 19600, + "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987654", + "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987", + [42], [["om", 100]], ["overflow-nested", -7] + ]] + ]], + [2, null], + [3, []] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(1, files.size()); + ASSERT_OK_AND_ASSIGN(auto meta, ReadShreddingMeta(files[0], "metrics", options)); + ASSERT_EQ(2, meta.num_columns); + ASSERT_EQ(3, meta.max_row_width); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [ + ["fixed-a", [ + true, 1, 2, 3, 4, 5.5, 6.25, "str", "bin", + "12345678.90", "123456789012345678.12345", 19500, + "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456789", + "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456", + [7, null, 8], [["m1", 10], ["m2", null]], ["nested", 99] + ]], + ["fixed-b", [ + null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null + ]], + ["overflow", [ + false, 9, 10, 11, 12, 13.5, 14.25, "overflow", "raw", + "-1.23", "-123456789012345678.12345", 19600, + "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987654", + "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987", + [42], [["om", 100]], ["overflow-nested", -7] + ]] + ]], + [0, 2, null], + [0, 3, []] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionReadFails) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto tag_value_type = arrow::struct_({ + arrow::field("v", arrow::int64()), + arrow::field("label", arrow::utf8()), + }); + auto profile_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int64()), + }); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), tag_value_type)), + arrow::field("profile", profile_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", [10, "one"]], ["z", [11, "overflow"]]], ["alice", 100]], + [2, [["a", [20, "two"]]], ["bob", 200]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", [10, "one"]], ["z", [11, "overflow"]]], ["alice", 100]], + [0, 2, [["a", [20, "two"]]], ["bob", 200]] + ])")); + ASSERT_TRUE(success); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(auto schema_v0, schema_manager.ReadSchema(0)); + std::vector fields_v0 = schema_v0->Fields(); + + auto read_fields = [&](const std::vector& field_names) -> Status { + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + (void)actual; + return Status::OK(); + }; + + auto tag_field = fields_v0[1].ArrowField(); + auto tag_map = arrow::internal::checked_pointer_cast(tag_field->type()); + auto tag_value_struct = + arrow::internal::checked_pointer_cast(tag_map->item_type()); + + // Simulate alter table changing the shared-shredding MAP value struct field type. + auto changed_tag_value_type = + arrow::map(tag_map->key_type(), tag_map->item_field()->WithType(arrow::struct_({ + tag_value_struct->field(0)->WithType(arrow::utf8()), + tag_value_struct->field(1), + }))); + std::vector fields_with_changed_tag_value = fields_v0; + fields_with_changed_tag_value[1] = + DataField(fields_v0[1].Id(), tag_field->WithType(changed_tag_value_type)); + ASSERT_OK(WriteNextSchema(fields_with_changed_tag_value, schema_v0->HighestFieldId(), options)); + ASSERT_NOK_WITH_MSG( + read_fields({"tags"}), + "PruneDataType nested item type mismatch inside map: read string vs data int64"); + + auto profile_field = fields_v0[2].ArrowField(); + auto profile_struct = + arrow::internal::checked_pointer_cast(profile_field->type()); + + // Simulate alter table renaming a nested field inside a STRUCT column. + std::vector fields_with_renamed_profile_child = fields_v0; + auto renamed_profile_type = arrow::struct_({ + profile_struct->field(0)->WithName("renamed_name"), + profile_struct->field(1), + }); + fields_with_renamed_profile_child[2] = + DataField(fields_v0[2].Id(), profile_field->WithType(renamed_profile_type)); + ASSERT_OK( + WriteNextSchema(fields_with_renamed_profile_child, schema_v0->HighestFieldId(), options)); + ASSERT_NOK_WITH_MSG(read_fields({"profile"}), + "name mismatch: read 'renamed_name' vs data 'name'"); + + // Simulate alter table changing a nested field type inside a STRUCT column. + std::vector fields_with_changed_profile_child_type = fields_v0; + auto changed_profile_type = arrow::struct_({ + profile_struct->field(0), + profile_struct->field(1)->WithType(arrow::utf8()), + }); + fields_with_changed_profile_child_type[2] = + DataField(fields_v0[2].Id(), profile_field->WithType(changed_profile_type)); + ASSERT_OK(WriteNextSchema(fields_with_changed_profile_child_type, schema_v0->HighestFieldId(), + options)); + ASSERT_NOK_WITH_MSG(read_fields({"profile"}), + "PruneDataType nested field type mismatch for 'score': read string vs " + "data int64"); +} + +// Keep ORC lazy dictionary decoding enabled across a default -> shared-shredding schema change. +// The test inspects every user-visible batch directly, because ReadResultCollector would otherwise +// decode dictionary arrays and hide a type mismatch between old and new files. +TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) { + auto [file_format, file_system] = GetParam(); + if (file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]]], + [2, [["a", "red"], ["z", "green"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(table_path, options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", "red"], ["z", "yellow"]]], + [4, [["a", "red"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", "red"], ["z", "blue"]]], + [0, 2, [["a", "red"], ["z", "green"]]], + [0, 3, [["a", "red"], ["z", "yellow"]]], + [0, 4, [["a", "red"]]] + ])")); + ASSERT_TRUE(success); +} + +// Verify shared-shredding in the PK read path. +TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + arrow::FieldVector fields = { + arrow::field("pk", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(auto batch_0, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, [["b", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto batch_1, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 100], ["z", 101]]], [3, [["c", 30]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 100], ["z", 101]]], + [0, 2, [["b", 20]]], + [0, 3, [["c", 30]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // max-columns=2, so keys are allocated in insertion order: + // Row 1: {a:1, b:2, c:3, d:4} -> physical cols: a, b; overflow: c, d + // Row 2: {a:10, b:20} -> physical cols: a, b; no overflow + // Row 3: null + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 1], ["b", 2], ["c", 3], ["d", 4]]], + [2, [["a", 10], ["b", 20]]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: read only key "a" (in physical column) + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1]]], + [0, 2, [["a", 10]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 2: read only key "c" (in overflow) + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"c"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["c", 3]]], + [0, 2, []], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 3: read keys "c" and "a" (cross physical + overflow), order follows user request + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"c,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["c", 3], ["a", 1]]], + [0, 2, [["a", 10]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissingKey) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // Row 1: tags=null + // Row 2: tags={b:2} (no key "a") + // Row 3: tags={a:30, b:40} + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, null], + [2, [["b", 2]]], + [3, [["a", 30], ["b", 40]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: row with null tags, request key "a" -> tags should be null + // Sub-case 2: row without key "a", request key "a" -> key "a" maps to null value + // Sub-case 3: row with key "a", request key "a" -> normal value + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, null], + [0, 2, []], + [0, 3, [["a", 30]]] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 4: request a key that was never written ("nonexistent") -> all rows have null value + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"nonexistent"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, null], + [0, 2, []], + [0, 3, []] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "3"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 1], ["b", 2]], [["x", 100], ["y", 200]]], + [2, [["a", 10], ["c", 30]], [["x", 1000], ["z", 3000]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: partial key recall on both columns independently + { + auto tags_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto metrics_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"x"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(tags_meta), + arrow::field("metrics", map_type)->WithMetadata(metrics_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1]], [["x", 100]]], + [0, 2, [["a", 10]], [["x", 1000]]] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 2: partial key recall on tags + full recall on metrics + { + auto tags_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a,b"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(tags_meta), + arrow::field("metrics", map_type), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1], ["b", 2]], [["x", 100], ["y", 200]]], + [0, 2, [["a", 10]], [["x", 1000], ["z", 3000]]] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartialKeyRecall) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 30], ["z", 31]]], + [4, [["z", 41]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options_v1, read_schema, expected_type, + R"([ + [0, 1, [["a", 10]]], + [0, 2, null], + [0, 3, [["a", 30]]], + [0, 4, []] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartialKeyRecall) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["z", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 30], ["z", 31]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options_v1, read_schema, expected_type, + R"([ + [0, 1, [["a", 10]]], + [0, 2, []], + [0, 3, [["a", 30]]], + [0, 4, null] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([[1, [["a", 10], ["b", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_NOK_WITH_MSG(ReadAndCheckWithReadSchema(options, read_schema, expected_type, "[]"), + "Duplicate selected key 'a'"); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) { + auto [file_format, file_system] = GetParam(); + if (file_format == "avro") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, null], + [2, null], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, null], + [0, 2, null], + [0, 3, null] + ])")); + ASSERT_TRUE(success); +} + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 62c61b7b..11564b93 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -277,6 +277,55 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface return array; } + SimpleStats GenerateBlobValueStats() const { + return BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, + std::vector({-1}), pool_.get()); + } + + Status CheckReadBlobs(const std::string& table_path, + const std::map& options, + const std::vector>& data_splits, + const std::string& blob_field, + const std::vector>& expected_blobs) const { + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); + PAIMON_ASSIGN_OR_RAISE(auto read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (read_result == nullptr) { + return Status::Invalid(fmt::format("No rows read for blob field {}", blob_field)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto combined_array, + arrow::Concatenate(read_result->chunks())); + auto struct_array = std::dynamic_pointer_cast(combined_array); + if (struct_array == nullptr) { + return Status::Invalid( + fmt::format("Read result for {} is not a struct array", blob_field)); + } + auto struct_type = std::dynamic_pointer_cast(struct_array->type()); + int blob_index = struct_type->GetFieldIndex(blob_field); + if (blob_index < 0) { + return Status::Invalid( + fmt::format("Blob field {} was not found in read result", blob_field)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto blob_struct_array, + arrow::StructArray::Make({struct_array->field(blob_index)}, + {BlobUtils::ToArrowField(blob_field, false)})); + PAIMON_ASSIGN_OR_RAISE( + auto actual_blobs, + TestHelper::ToBlobs(std::static_pointer_cast(blob_struct_array))); + PAIMON_ASSIGN_OR_RAISE(bool blobs_equal, TestHelper::CheckBlobsEqual( + actual_blobs, expected_blobs, file_system_)); + if (!blobs_equal) { + return Status::Invalid(fmt::format("Read blobs for {} do not match", blob_field)); + } + return Status::OK(); + } + void CheckCreationTime(const std::vector>& commit_messages) { TimezoneGuard guard("Asia/Shanghai"); for (const auto& msg : commit_messages) { @@ -296,11 +345,9 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface Status CommitMessages(const std::string& table_path, const std::vector>& commit_messages, - const std::map& commit_options, bool ignore_empty_commit = true, int64_t commit_identifier = BATCH_WRITE_COMMIT_IDENTIFIER) const { CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.SetOptions(commit_options); commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, @@ -412,8 +459,8 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -515,8 +562,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -585,8 +632,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -655,8 +702,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(8, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -720,8 +767,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(16, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(16, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 3); @@ -830,8 +877,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(6, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(6, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(6, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(6, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -893,8 +940,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(10, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1031,8 +1078,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round 1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1161,8 +1208,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round 2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1306,8 +1353,8 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_1, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -1450,8 +1497,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1554,8 +1601,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1663,8 +1710,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1737,8 +1784,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1795,8 +1842,8 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -1860,8 +1907,8 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -1876,6 +1923,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { } TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { + auto file_format = GetParam(); ::testing::GTEST_FLAG(throw_on_failure) = true; // create table arrow::FieldVector fields = { @@ -1884,7 +1932,6 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { auto schema = arrow::schema(fields); std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {"f1"}; - auto file_format = GetParam(); std::map options = { {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, @@ -1893,7 +1940,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { bool run_complete = false; auto io_hook = IOHook::GetInstance(); - for (size_t i = 0; i < 500; i++) { + // Loop bound must exceed the workflow's total IO operations so the loop can + // naturally terminate at the iteration where injection position falls past + // the last IO. Measured IO counts: orc=310, parquet=506, avro=195. + // 1000 leaves headroom for future format/workflow changes. + for (size_t i = 0; i < 1000; i++) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); @@ -2482,10 +2533,7 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { ASSERT_OK(file_store_write->Close()); // commit - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, commit_options, - /*ignore_empty_commit=*/false)); + ASSERT_OK(CommitMessages(table_path, commit_messages, /*ignore_empty_commit=*/false)); // check data file has field id meta std::vector> status_list; @@ -2567,10 +2615,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_EQ(results.size(), 1); auto commit_msg_impl = std::dynamic_pointer_cast(results[0]); auto meta = commit_msg_impl->data_increment_.new_files_[0]; - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); // check external path ASSERT_OK_AND_ASSIGN(bool file_exist, file_system_->Exists(meta->external_path.value())); @@ -2742,8 +2788,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -2811,8 +2857,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); std::string expected_data_2 = @@ -2888,10 +2934,8 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_EQ(results.size(), 3); auto commit_msg_impl = std::dynamic_pointer_cast(results[0]); auto meta = commit_msg_impl->data_increment_.new_files_[0]; - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -3073,10 +3117,8 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit()); ASSERT_EQ(results.size(), 1); - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); // check read result ScanContextBuilder scan_context_builder(table_path); @@ -3415,8 +3457,8 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -3751,20 +3793,26 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); int64_t commit_identifier = 0; std::vector> blob_descriptors; + std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; ASSERT_OK_AND_ASSIGN(auto blob1, Blob::FromPath(file1)); blob_descriptors.emplace_back(blob1->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob1))); std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(auto blob2, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob_descriptors.emplace_back(blob2->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob2))); ASSERT_OK_AND_ASSIGN(auto blob3, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); blob_descriptors.emplace_back(blob3->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob3))); ASSERT_OK_AND_ASSIGN(auto blob4, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); blob_descriptors.emplace_back(blob4->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob4))); std::vector>> blob_fields; blob_fields.emplace_back(std::move(blob_descriptors)); @@ -3790,9 +3838,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto file_meta2 = std::make_shared( "data-xxx.blob", /*file_size=*/764, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), @@ -3802,9 +3848,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto file_meta3 = std::make_shared( "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), @@ -3823,8 +3867,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); - ASSERT_EQ(8, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot.value().DeltaRecordCount()); ASSERT_EQ(4, snapshot.value().NextRowId().value()); // check data file meta after commit @@ -3836,6 +3880,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { for (size_t i = 0; i < expected_meta.size(); i++) { ASSERT_TRUE(data_split->DataFiles()[i]->TEST_Equal(*expected_meta[i])); } + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob", expected_blobs)); } TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { @@ -3894,8 +3939,8 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(2, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(2, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(2, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(2, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -3958,6 +4003,77 @@ TEST_P(WriteInteTest, TestNullabilityCheck) { /*expected_commit_messages=*/std::nullopt)); } +TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) { + auto file_format = GetParam(); + if (file_format == "avro") { + return; + } + + auto dir = UniqueTestDirectory::Create(); + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {Options::WRITE_BUFFER_SIZE, "1"}, + {Options::WRITE_BUFFER_SPILLABLE, "true"}, + {Options::WRITE_ONLY, "true"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "2"}, + }; + auto schema = arrow::schema(fields); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto table_path, CreateTestTable(dir->Str(), "db", "tbl", &c_schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, options)); + + std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp"); + WriteContextBuilder write_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_builder.SetOptions(options) + .WithStreamingMode(true) + .WithTempDirectory(tmp_dir) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + ASSERT_OK_AND_ASSIGN( + auto batch_0, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 10], ["b", 11], ["overflow", -10]]], [2, [["x", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(file_store_write->Write(std::move(batch_0))); + ASSERT_GT(TestHelper::CountChannelFiles(file_system_, tmp_dir), 0); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 100], ["b", 101], ["overflow", -100]]], [3, null]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(file_store_write->Write(std::move(batch_1))); + ASSERT_GT(TestHelper::CountChannelFiles(file_system_, tmp_dir), 0); + + ASSERT_OK_AND_ASSIGN(auto commit_messages, + file_store_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/0)); + ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); + ASSERT_OK(file_store_write->Close()); + + ASSERT_OK(ScanAndVerifyResult(table_path, fields, + R"([ + [0, 1, [["a", 100], ["b", 101], ["overflow", -100]]], + [0, 2, [["x", 20]]], + [0, 3, null] + ])")); +} + TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = { @@ -4009,9 +4125,7 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { file_store_write->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); std::string expected = R"([ @@ -4081,9 +4195,7 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); // Scan and verify both partitions @@ -4247,9 +4359,7 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { /*commit_identifier=*/0)); ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); // Scan: Alice deduped to f1=3, Bob f1=2 @@ -4346,9 +4456,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { /*commit_identifier=*/0)); // Spill files should be cleaned after PrepareCommit ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages_1, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages_1)); // Round 2: Bucket 0 writes Charlie + Bob(overwrite), Bucket 1 writes Frank + Eve(overwrite) auto r2_b0_batch1 = @@ -4379,7 +4487,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { /*commit_identifier=*/1)); // Spill files should be cleaned after PrepareCommit ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - ASSERT_OK(CommitMessages(table_path, commit_messages_2, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages_2)); ASSERT_OK(file_store_write->Close()); // Scan and verify per (partition, bucket) @@ -4523,10 +4631,8 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { io_hook->Clear(); // Commit both rounds - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(root_path, results_1.value(), pk_commit_options)); - ASSERT_OK(CommitMessages(root_path, results_2.value(), pk_commit_options)); + ASSERT_OK(CommitMessages(root_path, results_1.value())); + ASSERT_OK(CommitMessages(root_path, results_2.value())); ASSERT_OK(file_store_write->Close()); // Scan and verify final state after spill: @@ -4566,29 +4672,38 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); int64_t commit_identifier = 0; // Prepare blob descriptors for both blob fields std::vector> blob1_descriptors; std::vector> blob2_descriptors; + std::vector> expected_blob1s; + std::vector> expected_blob2s; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; ASSERT_OK_AND_ASSIGN(auto blob1_a, Blob::FromPath(file1)); blob1_descriptors.emplace_back(blob1_a->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_a))); std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(auto blob1_b, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob1_descriptors.emplace_back(blob1_b->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_b))); ASSERT_OK_AND_ASSIGN(auto blob1_c, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); blob1_descriptors.emplace_back(blob1_c->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_c))); // blob2 field uses different data slices ASSERT_OK_AND_ASSIGN(auto blob2_a, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); blob2_descriptors.emplace_back(blob2_a->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_a))); ASSERT_OK_AND_ASSIGN(auto blob2_b, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob2_descriptors.emplace_back(blob2_b->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_b))); ASSERT_OK_AND_ASSIGN(auto blob2_c, Blob::FromPath(file1)); blob2_descriptors.emplace_back(blob2_c->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_c))); std::vector>> blob_fields; blob_fields.emplace_back(std::move(blob1_descriptors)); @@ -4619,9 +4734,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto expected_blob1 = std::make_shared( "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), @@ -4633,9 +4746,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto expected_blob2 = std::make_shared( "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), @@ -4652,8 +4763,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); // 3 rows * 3 files (1 main + 1 blob1 + 1 blob2) = 9 total records - ASSERT_EQ(9, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(9, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot.value().TotalRecordCount()); + ASSERT_EQ(9, snapshot.value().DeltaRecordCount()); ASSERT_EQ(3, snapshot.value().NextRowId().value()); // Check data file meta after commit @@ -4678,6 +4789,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { } } } + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob1", expected_blob1s)); + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob2", expected_blob2s)); } TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommit) { diff --git a/test/test_data/cpp_tantivy_fixtures/english_default.archive b/test/test_data/cpp_tantivy_fixtures/english_default.archive new file mode 100644 index 00000000..d195af7e Binary files /dev/null and b/test/test_data/cpp_tantivy_fixtures/english_default.archive differ diff --git a/test/test_data/java_tantivy_fixtures/README.md b/test/test_data/java_tantivy_fixtures/README.md new file mode 100644 index 00000000..09361154 --- /dev/null +++ b/test/test_data/java_tantivy_fixtures/README.md @@ -0,0 +1,51 @@ +# Java -> C++ tantivy cross-read fixtures + +> Generated on **2026-04-23** for `paimon-tantivy-java-compat-test`. + +## Contents + +| File | Purpose | +|---|---| +| `english_simple.archive` | A BE archive produced by paimon-java's `TantivyIndexWriter + packIndex` path; 10 plain-English documents, row_ids 0..9 | +| `english_simple.golden.json` | Human-readable golden file: expected row_ids for each query type | + +## Pinned versions + +| Component | Version | +|---|---| +| tantivy crate | **0.22.1** | +| paimon-tantivy-jni | latest git sha at generation time (commit lives in the paimon repo) | +| schema | `row_id` u64 stored+indexed+fast + `text` TEXT | +| archive byte format | Java-compatible, big-endian, no version header | + +Upgrading any component (especially the **tantivy version**) can make the segment +files binary-incompatible — regenerate the fixtures: + +```bash +# 1. Build the Java native lib (if the Rust side changed) +cd /path/to/paimon/paimon-tantivy/paimon-tantivy-jni/rust && cargo build --release +cp target/release/libtantivy_jni.dylib \ + ../src/main/resources/native/darwin-aarch64/ + +# 2. mvn install + run the fixture generator +cd /path/to/paimon +mvn install -pl paimon-tantivy/paimon-tantivy-index -am -DskipTests -Denforcer.skip=true +mvn -pl paimon-tantivy/paimon-tantivy-index test \ + -Dtest=TantivyIndexFixtureGen -DfailIfNoTests=false \ + -Denforcer.skip=true \ + -DfixtureOutDir=/path/to/paimon-cpp/test/test_data/java_tantivy_fixtures +``` + +## Verification + +``` +xxd english_simple.archive | head -1 +# 00000000: 00 00 00 16 ... <- BE int32 file_count = 22 (Java does not +# force-merge, so multiple segments) +``` + +## Related code + +- [`tantivy_java_compat_test.cpp`](../../../src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp) +- [`tantivy CMakeLists.txt`](../../../src/paimon/global_index/tantivy/CMakeLists.txt) +- [`tantivy_ffi writer.rs`](../../../crates/tantivy_ffi/src/writer.rs) diff --git a/test/test_data/java_tantivy_fixtures/english_simple.archive b/test/test_data/java_tantivy_fixtures/english_simple.archive new file mode 100644 index 00000000..c0849957 Binary files /dev/null and b/test/test_data/java_tantivy_fixtures/english_simple.archive differ diff --git a/test/test_data/java_tantivy_fixtures/english_simple.golden.json b/test/test_data/java_tantivy_fixtures/english_simple.golden.json new file mode 100644 index 00000000..a3c12060 --- /dev/null +++ b/test/test_data/java_tantivy_fixtures/english_simple.golden.json @@ -0,0 +1,25 @@ +{ + "description": "10 English docs; row_ids 0..9; generated by TantivyIndexFixtureGen via TantivyFullTextGlobalIndexWriter production path; consumed by the paimon-cpp cross-read test.", + "docs": [ + {"row_id": 0, "text": "apple banana cherry"}, + {"row_id": 1, "text": "apple durian"}, + {"row_id": 2, "text": "banana cherry"}, + {"row_id": 3, "text": "fig grape"}, + {"row_id": 4, "text": "apple cherry fig"}, + {"row_id": 5, "text": "banana elderberry"}, + {"row_id": 6, "text": "cherry durian"}, + {"row_id": 7, "text": "apple"}, + {"row_id": 8, "text": "grape fig elderberry"}, + {"row_id": 9, "text": "cherry fig"} + ], + "queries": [ + {"type": "match_all", "query": "apple", "expected_row_ids": [0, 1, 4, 7]}, + {"type": "match_all", "query": "apple banana", "expected_row_ids": [0]}, + {"type": "match_any", "query": "durian elderberry", "expected_row_ids": [1, 5, 6, 8]}, + {"type": "phrase", "query": "apple banana", "expected_row_ids": [0]}, + {"type": "phrase", "query": "banana cherry", "expected_row_ids": [0, 2]}, + {"type": "prefix", "query": "ap", "expected_row_ids": [0, 1, 4, 7]}, + {"type": "wildcard", "query": "*err*", "expected_row_ids": [0, 2, 4, 5, 6, 8, 9]}, + {"type": "match_any", "query": "apple banana cherry durian fig grape elderberry", "expected_row_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} + ] +} diff --git a/test/test_data/java_tantivy_fixtures/production_sample.archive b/test/test_data/java_tantivy_fixtures/production_sample.archive new file mode 100644 index 00000000..0f829718 Binary files /dev/null and b/test/test_data/java_tantivy_fixtures/production_sample.archive differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README index 9f3f85b1..3e17ccd4 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README @@ -42,8 +42,7 @@ renameColumn("f3", "f0") renameColumn("f5", "f3") updateColumnPosition(SchemaChange.Move.first("f4")) addColumn("f6", INT()) -renameColumn("blob", "f5") -updateColumnPosition(SchemaChange.Move.after("f5", "f2")) +updateColumnPosition(SchemaChange.Move.after("blob", "f2")) f4:TIMESTAMP(9):6 key0:INT:0 @@ -51,7 +50,7 @@ key1:INT:1 f3:INT:2 f1:STRING:3 f2:DECIMAL(6, 3):4 -f5:BLOB NOT NULL:8 +blob:BLOB NOT NULL:8 f0:BOOLEAN:5 f6:INT:9 @@ -66,7 +65,7 @@ set first row id to 5 commit snapshot-2 NoCompact -write "f4", "key0", "key1", "f2", "f0", "f6", "f5" +write "f4", "key0", "key1", "f2", "f0", "f6", "blob" Add(Timestamp(1732603136054l, 154), 0, 1, "55.002", true, 56, "Apple") Add(Timestamp(1732603136064l, 164), 0, 1, "666.012", false, 66, "Banana") Add(Timestamp(1732603136074l, 174), 0, 1, "-77.022", true, 76, "Cherry") @@ -78,7 +77,7 @@ commit snapshot-3 NoCompact Recall with schema-1, with _ROW_ID and _SEQUENCE_NUMBER, result: -[f4, key0, key1, f3, f1, f2, f5, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] +[f4, key0, key1, f3, f1, f2, blob, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] [TIMESTAMP(9), INT, INT, INT, STRING, DECIMAL(6, 3), BLOB NOT NULL, BOOLEAN, INT, BIGINT, BIGINT NOT NULL] INSERT: 1970-01-05T00:00, 0, 1, 100, 2024-11-26 06:38:56.001000001, 0.020, Lily, true, NULL, 0, 1 INSERT: 1969-11-18T00:00, 0, 1, 110, 2024-11-26 06:38:56.011000011, 11.120, Alice, true, NULL, 1, 1 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-17cd7b66-4ca1-42b3-a569-d7b0b889d314-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2c772a51-21ba-4c17-b464-6f1a314d0950-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-17cd7b66-4ca1-42b3-a569-d7b0b889d314-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2c772a51-21ba-4c17-b464-6f1a314d0950-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-1.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-1.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-1.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-1.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-2.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-2.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-2.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-2.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-1.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-1.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-2.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-2.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-35fe228b-1e40-44dd-9832-ff078fc60150-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-d0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-35fe228b-1e40-44dd-9832-ff078fc60150-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-d0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 deleted file mode 100644 index ede0fd39..00000000 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 and /dev/null differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 new file mode 100644 index 00000000..d597e233 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-c1e6e601-fc3b-4ab1-8b53-dd661f91a59c-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-c1e6e601-fc3b-4ab1-8b53-dd661f91a59c-0 new file mode 100644 index 00000000..54d935e4 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-c1e6e601-fc3b-4ab1-8b53-dd661f91a59c-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 new file mode 100644 index 00000000..57e6edf4 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 new file mode 100644 index 00000000..7964d271 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-1 new file mode 100644 index 00000000..6e191db2 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-1 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 new file mode 100644 index 00000000..04ebc021 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-1 new file mode 100644 index 00000000..7964d271 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-1 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-0 new file mode 100644 index 00000000..2ed0c9c3 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 new file mode 100644 index 00000000..0091f8e7 Binary files /dev/null and b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 deleted file mode 100644 index 12027972..00000000 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 and /dev/null differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 deleted file mode 100644 index 6e740fbd..00000000 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 and /dev/null differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 index 6fdd4b85..5dae4544 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "data-evolution.enabled" : "true", - "manifest.format" : "avro", + "bucket" : "-1", "blob.target-file-size" : "50", - "file.format" : "orc", - "row-tracking.enabled" : "true" + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "orc", + "file.format" : "orc" }, - "timeMillis" : 1762240062329 + "timeMillis" : 1780565151810 } \ No newline at end of file diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 index 9677faa5..194fb626 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 @@ -27,7 +27,7 @@ "type" : "DECIMAL(6, 3)" }, { "id" : 8, - "name" : "f5", + "name" : "blob", "type" : "BLOB NOT NULL" }, { "id" : 5, @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { - "data-evolution.enabled" : "true", - "manifest.format" : "avro", + "bucket" : "-1", "blob.target-file-size" : "50", - "file.format" : "orc", - "row-tracking.enabled" : "true" + "row-tracking.enabled" : "true", + "data-evolution.enabled" : "true", + "manifest.format" : "orc", + "file.format" : "orc" }, - "timeMillis" : 1762240063751 + "timeMillis" : 1780565153189 } \ No newline at end of file diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 index dc479f2e..ed3c4054 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 @@ -2,18 +2,15 @@ "version" : 3, "id" : 1, "schemaId" : 0, - "baseManifestList" : "manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0", - "baseManifestListSize" : 884, - "deltaManifestList" : "manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-1", - "deltaManifestListSize" : 995, - "changelogManifestList" : null, - "commitUser" : "1f85e3d0-2588-469c-9cfc-b15b05478937", - "commitIdentifier" : 1, + "baseManifestList" : "manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0", + "baseManifestListSize" : 392, + "deltaManifestList" : "manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-1", + "deltaManifestListSize" : 1543, + "commitUser" : "7476fefe-afad-457c-b3e8-3e56cc1a4856", + "commitIdentifier" : 0, "commitKind" : "APPEND", - "timeMillis" : 1762240063499, - "logOffsets" : { }, + "timeMillis" : 1780565153104, "totalRecordCount" : 15, "deltaRecordCount" : 15, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 index 62842ba1..a34213f6 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 @@ -2,18 +2,15 @@ "version" : 3, "id" : 2, "schemaId" : 1, - "baseManifestList" : "manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0", - "baseManifestListSize" : 995, - "deltaManifestList" : "manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1", - "deltaManifestListSize" : 996, - "changelogManifestList" : null, - "commitUser" : "c994cddc-02cc-478b-8b1f-37fe48574b31", - "commitIdentifier" : 2, + "baseManifestList" : "manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0", + "baseManifestListSize" : 1543, + "deltaManifestList" : "manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-1", + "deltaManifestListSize" : 1565, + "commitUser" : "be4ac30f-72d8-4764-a282-55b1d39e1027", + "commitIdentifier" : 1, "commitKind" : "APPEND", - "timeMillis" : 1762240063850, - "logOffsets" : { }, + "timeMillis" : 1780565153415, "totalRecordCount" : 20, "deltaRecordCount" : 5, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 index 54b1bdab..0a262914 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 @@ -2,18 +2,15 @@ "version" : 3, "id" : 3, "schemaId" : 1, - "baseManifestList" : "manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2", - "baseManifestListSize" : 1034, - "deltaManifestList" : "manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3", - "deltaManifestListSize" : 996, - "changelogManifestList" : null, - "commitUser" : "c994cddc-02cc-478b-8b1f-37fe48574b31", - "commitIdentifier" : 3, + "baseManifestList" : "manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-0", + "baseManifestListSize" : 1723, + "deltaManifestList" : "manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1", + "deltaManifestListSize" : 1569, + "commitUser" : "40830793-1dbe-4e84-abfc-1d2e6889880c", + "commitIdentifier" : 2, "commitKind" : "APPEND", - "timeMillis" : 1762240063892, - "logOffsets" : { }, + "timeMillis" : 1780565153586, "totalRecordCount" : 30, "deltaRecordCount" : 10, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/README b/test/test_data/orc/blob_desc_field.db/blob_desc_field/README new file mode 100644 index 00000000..7d923daf --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/README @@ -0,0 +1,26 @@ +f0:int b0:blob b1:blob b2:blob b3:blob (all can be null) +bucket count: -1 +target-file-size: 700 +row-tracking.enabled: true +data-evolution.enabled: true +blob-descriptor-field: b0,b1 + +b0: descriptor field, inline in main file, source .bin files in raw_blob/ +b1: descriptor field whose descriptor points to external_blob/ +b2: regular blob, written to .blob files +b3: regular blob, written to .blob files + +Note: this fixture keeps historical descriptor URIs pointing at raw_blob/ and external_blob/. + +Msgs: +snapshot-1 +write field: "f0", "b0", "b1", "b2", "b3" +Add: 1, "img_0", null, "raw_2_0", "raw_3_0" +Add: 2, "img_1", "vid_1", "raw_2_1", null +Add: 3, "img_2", null, "raw_2_2", "raw_3_2" +NoCompact + +C++ read note: + Descriptor URIs contain absolute paths from the Java generation machine. + ConvertDescriptorToRawBlob uses BlobDescriptorPathRewrite{"raw_blob", "external_blob"} + to redirect them to
/raw_blob/ and
/external_blob/ at read time. diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc new file mode 100644 index 00000000..179e5412 Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob new file mode 100644 index 00000000..9cbdf5db Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob new file mode 100644 index 00000000..8e8a3cdb Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob new file mode 100644 index 00000000..fa69a509 Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 new file mode 100644 index 00000000..5a29f6c0 Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 new file mode 100644 index 00000000..04ebc021 Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 new file mode 100644 index 00000000..0737192c Binary files /dev/null and b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 differ diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin new file mode 100644 index 00000000..1207296c --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin @@ -0,0 +1 @@ +img_0 \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin new file mode 100644 index 00000000..a3f12b37 --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin @@ -0,0 +1 @@ +img_1 \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin new file mode 100644 index 00000000..c4c94eef --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin @@ -0,0 +1 @@ +img_2 \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 new file mode 100644 index 00000000..d013d274 --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 @@ -0,0 +1,39 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "f0", + "type" : "INT" + }, { + "id" : 1, + "name" : "b0", + "type" : "BLOB" + }, { + "id" : 2, + "name" : "b1", + "type" : "BLOB" + }, { + "id" : 3, + "name" : "b2", + "type" : "BLOB" + }, { + "id" : 4, + "name" : "b3", + "type" : "BLOB" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "row-tracking.enabled" : "true", + "target-file-size" : "700", + "data-evolution.enabled" : "true", + "file-system" : "local", + "manifest.format" : "orc", + "file.format" : "orc", + "blob-descriptor-field" : "b0,b1" + }, + "timeMillis" : 1781088844975 +} diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/LATEST b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/LATEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 new file mode 100644 index 00000000..92a0f00f --- /dev/null +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 @@ -0,0 +1,16 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0", + "baseManifestListSize" : 392, + "deltaManifestList" : "manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1", + "deltaManifestListSize" : 1510, + "commitUser" : "f69bd0bf-9271-465c-a6ea-60001dfe02ff", + "commitIdentifier" : 0, + "commitKind" : "APPEND", + "timeMillis" : 1781088845796, + "totalRecordCount" : 9, + "deltaRecordCount" : 9, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README index 9f3f85b1..3e17ccd4 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README @@ -42,8 +42,7 @@ renameColumn("f3", "f0") renameColumn("f5", "f3") updateColumnPosition(SchemaChange.Move.first("f4")) addColumn("f6", INT()) -renameColumn("blob", "f5") -updateColumnPosition(SchemaChange.Move.after("f5", "f2")) +updateColumnPosition(SchemaChange.Move.after("blob", "f2")) f4:TIMESTAMP(9):6 key0:INT:0 @@ -51,7 +50,7 @@ key1:INT:1 f3:INT:2 f1:STRING:3 f2:DECIMAL(6, 3):4 -f5:BLOB NOT NULL:8 +blob:BLOB NOT NULL:8 f0:BOOLEAN:5 f6:INT:9 @@ -66,7 +65,7 @@ set first row id to 5 commit snapshot-2 NoCompact -write "f4", "key0", "key1", "f2", "f0", "f6", "f5" +write "f4", "key0", "key1", "f2", "f0", "f6", "blob" Add(Timestamp(1732603136054l, 154), 0, 1, "55.002", true, 56, "Apple") Add(Timestamp(1732603136064l, 164), 0, 1, "666.012", false, 66, "Banana") Add(Timestamp(1732603136074l, 174), 0, 1, "-77.022", true, 76, "Cherry") @@ -78,7 +77,7 @@ commit snapshot-3 NoCompact Recall with schema-1, with _ROW_ID and _SEQUENCE_NUMBER, result: -[f4, key0, key1, f3, f1, f2, f5, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] +[f4, key0, key1, f3, f1, f2, blob, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] [TIMESTAMP(9), INT, INT, INT, STRING, DECIMAL(6, 3), BLOB NOT NULL, BOOLEAN, INT, BIGINT, BIGINT NOT NULL] INSERT: 1970-01-05T00:00, 0, 1, 100, 2024-11-26 06:38:56.001000001, 0.020, Lily, true, NULL, 0, 1 INSERT: 1969-11-18T00:00, 0, 1, 110, 2024-11-26 06:38:56.011000011, 11.120, Alice, true, NULL, 1, 1 diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-0.parquet new file mode 100644 index 00000000..5819be52 Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-0.parquet differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-1.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-1.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-1.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-1.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-2.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-2.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-2.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-2.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet new file mode 100644 index 00000000..307553da Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet deleted file mode 100644 index 9fdb6073..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet new file mode 100644 index 00000000..35224b8e Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-1.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-1.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-2.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-2.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet deleted file mode 100644 index e855b6c3..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-0.parquet deleted file mode 100644 index 1611b36b..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-0.parquet and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-f9d44206-06f9-4411-b69f-749f7868ac2d-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-f9d44206-06f9-4411-b69f-749f7868ac2d-0.parquet deleted file mode 100644 index 3a4d5e51..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-f9d44206-06f9-4411-b69f-749f7868ac2d-0.parquet and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet new file mode 100644 index 00000000..04132a79 Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-04dfe945-c569-4a24-b830-c69d4f426e64-0 similarity index 88% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-04dfe945-c569-4a24-b830-c69d4f426e64-0 index 8f8263d5..22e9496d 100644 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-0 and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-04dfe945-c569-4a24-b830-c69d4f426e64-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0 similarity index 84% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0 index 2407d6d0..b7c6ea66 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 deleted file mode 100644 index f65b6ddd..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0 similarity index 86% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0 index 480e49b9..7a696089 100644 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 deleted file mode 100644 index 1179a001..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 deleted file mode 100644 index 48b19be2..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2 deleted file mode 100644 index d26360e8..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3 deleted file mode 100644 index 33779160..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3 and /dev/null differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0 similarity index 68% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0 index 14070357..1404cbe4 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1 new file mode 100644 index 00000000..ee129c72 Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0 similarity index 76% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0 index cc1b1ddd..1b555e96 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1 new file mode 100644 index 00000000..41dd0a5a Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 new file mode 100644 index 00000000..e03501c3 Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1 new file mode 100644 index 00000000..97c65ed9 Binary files /dev/null and b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1 differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0 deleted file mode 100644 index 3102f50a..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1 deleted file mode 100644 index f720ef96..00000000 Binary files a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1 and /dev/null differ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 index 5f36c97d..42062ec2 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { + "bucket" : "-1", + "blob.target-file-size" : "50", + "row-tracking.enabled" : "true", "data-evolution.enabled" : "true", "manifest.format" : "avro", - "blob.target-file-size" : "50", - "file.format" : "parquet", - "row-tracking.enabled" : "true" + "file.format" : "parquet" }, - "timeMillis" : 1762240078799 + "timeMillis" : 1780568705752 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 index e2f4b655..35f77e5c 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 @@ -27,7 +27,7 @@ "type" : "DECIMAL(6, 3)" }, { "id" : 8, - "name" : "f5", + "name" : "blob", "type" : "BLOB NOT NULL" }, { "id" : 5, @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { + "bucket" : "-1", + "blob.target-file-size" : "50", + "row-tracking.enabled" : "true", "data-evolution.enabled" : "true", "manifest.format" : "avro", - "blob.target-file-size" : "50", - "file.format" : "parquet", - "row-tracking.enabled" : "true" + "file.format" : "parquet" }, - "timeMillis" : 1762240081035 + "timeMillis" : 1780568707875 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 index 02a8cd25..4a38b005 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 @@ -2,18 +2,15 @@ "version" : 3, "id" : 1, "schemaId" : 0, - "baseManifestList" : "manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0", - "baseManifestListSize" : 884, - "deltaManifestList" : "manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1", - "deltaManifestListSize" : 995, - "changelogManifestList" : null, - "commitUser" : "ec4202d2-8ff3-4c75-9e6a-23281dad6562", - "commitIdentifier" : 1, + "baseManifestList" : "manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1", + "deltaManifestListSize" : 1119, + "commitUser" : "7aec5357-eb8d-42b4-bb9e-1753ec6e10ae", + "commitIdentifier" : 0, "commitKind" : "APPEND", - "timeMillis" : 1762240080710, - "logOffsets" : { }, + "timeMillis" : 1780568707810, "totalRecordCount" : 15, "deltaRecordCount" : 15, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 index 76fdce1b..3ea1d014 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 @@ -2,18 +2,15 @@ "version" : 3, "id" : 2, "schemaId" : 1, - "baseManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0", - "baseManifestListSize" : 995, - "deltaManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1", - "deltaManifestListSize" : 996, - "changelogManifestList" : null, - "commitUser" : "9a49e1a1-9141-41a4-bfd1-403f0679d72c", - "commitIdentifier" : 2, + "baseManifestList" : "manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1", + "deltaManifestListSize" : 1121, + "commitUser" : "732dc057-d23c-456e-96a7-b888244afe95", + "commitIdentifier" : 1, "commitKind" : "APPEND", - "timeMillis" : 1762240081198, - "logOffsets" : { }, + "timeMillis" : 1780568707986, "totalRecordCount" : 20, "deltaRecordCount" : 5, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 index cf9eb13b..3c972724 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 @@ -2,18 +2,15 @@ "version" : 3, "id" : 3, "schemaId" : 1, - "baseManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2", - "baseManifestListSize" : 1032, - "deltaManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3", - "deltaManifestListSize" : 997, - "changelogManifestList" : null, - "commitUser" : "9a49e1a1-9141-41a4-bfd1-403f0679d72c", - "commitIdentifier" : 3, + "baseManifestList" : "manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0", + "baseManifestListSize" : 1155, + "deltaManifestList" : "manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1", + "deltaManifestListSize" : 1124, + "commitUser" : "6ae1e966-c41a-4699-b01f-0caf0b76c924", + "commitIdentifier" : 2, "commitKind" : "APPEND", - "timeMillis" : 1762240081236, - "logOffsets" : { }, + "timeMillis" : 1780568708049, "totalRecordCount" : 30, "deltaRecordCount" : 10, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/README b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/README new file mode 100644 index 00000000..7d923daf --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/README @@ -0,0 +1,26 @@ +f0:int b0:blob b1:blob b2:blob b3:blob (all can be null) +bucket count: -1 +target-file-size: 700 +row-tracking.enabled: true +data-evolution.enabled: true +blob-descriptor-field: b0,b1 + +b0: descriptor field, inline in main file, source .bin files in raw_blob/ +b1: descriptor field whose descriptor points to external_blob/ +b2: regular blob, written to .blob files +b3: regular blob, written to .blob files + +Note: this fixture keeps historical descriptor URIs pointing at raw_blob/ and external_blob/. + +Msgs: +snapshot-1 +write field: "f0", "b0", "b1", "b2", "b3" +Add: 1, "img_0", null, "raw_2_0", "raw_3_0" +Add: 2, "img_1", "vid_1", "raw_2_1", null +Add: 3, "img_2", null, "raw_2_2", "raw_3_2" +NoCompact + +C++ read note: + Descriptor URIs contain absolute paths from the Java generation machine. + ConvertDescriptorToRawBlob uses BlobDescriptorPathRewrite{"raw_blob", "external_blob"} + to redirect them to
/raw_blob/ and
/external_blob/ at read time. diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet new file mode 100644 index 00000000..42bd60e0 Binary files /dev/null and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet differ diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob new file mode 100644 index 00000000..9cbdf5db Binary files /dev/null and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob differ diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob new file mode 100644 index 00000000..8e8a3cdb Binary files /dev/null and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob differ diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob new file mode 100644 index 00000000..fa69a509 Binary files /dev/null and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 similarity index 88% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 index c34df04b..e8c82cc3 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-1 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 similarity index 76% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-1 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 index cacd0b15..402ee39d 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-1 and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 differ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 similarity index 68% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 index 51b3950c..48db5679 100644 Binary files a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 and b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 differ diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin new file mode 100644 index 00000000..1207296c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin @@ -0,0 +1 @@ +img_0 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin new file mode 100644 index 00000000..a3f12b37 --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin @@ -0,0 +1 @@ +img_1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin new file mode 100644 index 00000000..c4c94eef --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin @@ -0,0 +1 @@ +img_2 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 new file mode 100644 index 00000000..0c314d4f --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 @@ -0,0 +1,39 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "f0", + "type" : "INT" + }, { + "id" : 1, + "name" : "b0", + "type" : "BLOB" + }, { + "id" : 2, + "name" : "b1", + "type" : "BLOB" + }, { + "id" : 3, + "name" : "b2", + "type" : "BLOB" + }, { + "id" : 4, + "name" : "b3", + "type" : "BLOB" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "row-tracking.enabled" : "true", + "target-file-size" : "700", + "data-evolution.enabled" : "true", + "file-system" : "local", + "manifest.format" : "avro", + "file.format" : "parquet", + "blob-descriptor-field" : "b0,b1" + }, + "timeMillis" : 1781088620905 +} diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/LATEST b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/LATEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 new file mode 100644 index 00000000..99789afc --- /dev/null +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 @@ -0,0 +1,16 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1", + "deltaManifestListSize" : 1115, + "commitUser" : "d30e72bf-d9bb-4de4-a178-f11f2d2f4fa5", + "commitIdentifier" : 0, + "commitKind" : "APPEND", + "timeMillis" : 1781088622173, + "totalRecordCount" : 9, + "deltaRecordCount" : 9, + "nextRowId" : 3 +} \ No newline at end of file diff --git a/test/test_data/tokenizer_golden/README.md b/test/test_data/tokenizer_golden/README.md new file mode 100644 index 00000000..b6ba3d20 --- /dev/null +++ b/test/test_data/tokenizer_golden/README.md @@ -0,0 +1,29 @@ +# Tokenizer golden samples + +Used by `paimon-tantivy-tokenizer-test` to compare cppjieba vs jieba-rs +tokenization output. + +## Files + +- `golden_synthetic.txt` — hand-written edge cases (mixed Chinese/English, + digits, punctuation, emoji, whitespace, very long words, ...) +- `golden_corpus.txt` — short excerpts from public corpora (general knowledge, + no copyright concerns) + +## Usage + +The test code (see `src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp`): +1. reads the files line by line +2. tokenizes each line with cppjieba `JiebaTokenizer::CutWithMode` + `Normalize` + to get token sequence A +3. tokenizes each line with the jieba-rs FFI `paimon_tantivy_tokenizer_tokenize` + to get token sequence B +4. compares A and B: the line passes if they are identical, otherwise it is + recorded in the diff report +5. (historical) the original acceptance bar was a diff rate <= 1%; the test is + now advisory only and logs diffs without failing + +## Extending + +To add business query logs later, drop a new `golden_business.txt` in this +directory; the test scans `golden_*.txt` automatically. diff --git a/test/test_data/tokenizer_golden/golden_corpus.txt b/test/test_data/tokenizer_golden/golden_corpus.txt new file mode 100644 index 00000000..38c7c887 --- /dev/null +++ b/test/test_data/tokenizer_golden/golden_corpus.txt @@ -0,0 +1,20 @@ +人工智能是计算机科学的一个分支 +机器学习是人工智能的核心领域 +深度学习使用神经网络进行模式识别 +大语言模型基于 Transformer 架构 +开源软件促进了全球技术合作 +Rust 语言以内存安全著称 +Python 广泛应用于数据科学 +分布式系统需要处理网络分区问题 +数据库事务保证原子性一致性隔离性持久性 +编程的艺术在于解决复杂问题 +搜索引擎依赖倒排索引加速查询 +自然语言处理技术日新月异 +云计算降低了基础设施成本 +开发者社区推动了技术进步 +版本控制系统是协作的基石 +操作系统管理计算机的硬件资源 +编译器将源代码翻译成机器指令 +算法的时间复杂度决定了执行效率 +数据结构的选择影响程序性能 +网络协议定义了通信的规则 diff --git a/test/test_data/tokenizer_golden/golden_synthetic.txt b/test/test_data/tokenizer_golden/golden_synthetic.txt new file mode 100644 index 00000000..65b14474 --- /dev/null +++ b/test/test_data/tokenizer_golden/golden_synthetic.txt @@ -0,0 +1,38 @@ +Hello World +hello world +HELLO WORLD +Hello 世界 +你好世界 +中国人民共和国 +我爱北京天安门 +北京是中华人民共和国的首都 +南京市长江大桥 +他来到了网易杭研大厦 +小明硕士毕业于中国科学院计算所,后在日本京都大学深造 +工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作 +结婚的和尚未结婚的 +程序员用Python和Rust写代码 +this is a test 这是一个测试 +Rust tantivy 全文索引 +C++ 到 Rust 的 FFI 桥接 +cpp cppjieba jieba-rs +分词器 tokenizer +全文 search +倒排索引 inverted index +paimon-cpp tantivy-fts +100个中文字符被分词器处理 +超长词最长词最长词最长词最长词最长词最长词 +... +!@#$%^&*() +"hello" +'quoted' +content +{json: "value"} +[1,2,3] +line1 +line2 +CJK 标点、。！？ +全角：ＡＢＣ１２３ +ABC123 混合数字字母 +abc123 +ABC123 diff --git a/test/test_data/tokenizer_golden/known_diffs.txt b/test/test_data/tokenizer_golden/known_diffs.txt new file mode 100644 index 00000000..23073bd3 --- /dev/null +++ b/test/test_data/tokenizer_golden/known_diffs.txt @@ -0,0 +1,18 @@ +abc_123 +foo.bar.baz +https://example.com/path?q=1 +email@example.com +192.168.1.1 +2026-04-20 +12:34:56 +$100 ¥200 €300 +100% +3.14 +-1 -2 -3 +a b c d e + + tab tab +mixed space tab +空 白 和 tab + leading and trailing +中英混合 Mixed CN EN diff --git a/third_party/jindosdk-nextarch/CMakeLists.txt b/third_party/jindosdk-nextarch/CMakeLists.txt index 1f921097..2ff20461 100644 --- a/third_party/jindosdk-nextarch/CMakeLists.txt +++ b/third_party/jindosdk-nextarch/CMakeLists.txt @@ -28,7 +28,8 @@ find_package(jindosdk_c REQUIRED) file(GLOB_RECURSE JINDOSDK_SOURCES src/*.cpp) add_library(jindosdk-nextarch STATIC ${JINDOSDK_SOURCES}) -target_link_libraries(jindosdk-nextarch PUBLIC pthread dl JINDOSDK::JINDOSDK) +target_link_libraries(jindosdk-nextarch PUBLIC pthread ${CMAKE_DL_LIBS} + JINDOSDK::JINDOSDK) target_include_directories(jindosdk-nextarch PUBLIC $ $) diff --git a/third_party/versions.txt b/third_party/versions.txt index a49a128a..d27b9e81 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -60,6 +60,10 @@ PAIMON_GTEST_BUILD_VERSION=1.11.0 PAIMON_GTEST_BUILD_SHA256_CHECKSUM=b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5 PAIMON_GTEST_PKG_NAME=gtest-${PAIMON_GTEST_BUILD_VERSION}.tar.gz +PAIMON_BENCHMARK_BUILD_VERSION=1.9.1 +PAIMON_BENCHMARK_BUILD_SHA256_CHECKSUM=32131c08ee31eeff2c8968d7e874f3cb648034377dfc32a4c377fa8796d84981 +PAIMON_BENCHMARK_PKG_NAME=benchmark-${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz + PAIMON_ARROW_BUILD_VERSION=17.0.0 PAIMON_ARROW_BUILD_SHA256_CHECKSUM=9d280d8042e7cf526f8c28d170d93bfab65e50f94569f6a790982a878d8d898d PAIMON_ARROW_PKG_NAME=apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz @@ -68,6 +72,31 @@ PAIMON_AVRO_BUILD_VERSION=c499eefb48aa2db906c7bca14a047223806f36db PAIMON_AVRO_BUILD_SHA256_CHECKSUM=9771f1dcfe3c01aff7ff670e873e66d3406362f71941821d482de65f3d32d780 PAIMON_AVRO_PKG_NAME=avro-${PAIMON_AVRO_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_AUTH_BUILD_VERSION=v0.10.3 +PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=20fc5e75529fadd81fd38b25f9d83798b53ab235ebbac92cdfbb716cfcc7593d +PAIMON_AWS_C_AUTH_PKG_NAME=aws-c-auth-${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_CAL_BUILD_VERSION=v0.9.14 +PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM=0e96e0067fa921768e07b5b4ebad82011ccf474903e9286419ef428d68f317ea +PAIMON_AWS_C_CAL_PKG_NAME=aws-c-cal-${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_COMMON_BUILD_VERSION=v0.14.0 +PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=3684076ec5da899074336722ba58a01f7166a1a2e5ad72f846f6fd468ecdf2ec +PAIMON_AWS_C_COMMON_PKG_NAME=aws-c-common-${PAIMON_AWS_C_COMMON_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.2 +PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=f93f5a5d8b3fee3a6d97b14ba279efacd4d4016ef9cc7dc4be7d43519ecfbe93 +PAIMON_AWS_C_COMPRESSION_PKG_NAME=aws-c-compression-${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_HTTP_BUILD_VERSION=v0.11.0 +PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=4ccbdd33c798b590288330dec9e93abe2ff6cfb198b7a4db036c9d362f2e6506 +PAIMON_AWS_C_HTTP_PKG_NAME=aws-c-http-${PAIMON_AWS_C_HTTP_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_IO_BUILD_VERSION=v0.27.0 +PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM=e89a1f784e7c97e4197031ffdcf30f67d66d7c14f8a391edf5764f17dae982ee +PAIMON_AWS_C_IO_PKG_NAME=aws-c-io-${PAIMON_AWS_C_IO_BUILD_VERSION}.tar.gz +PAIMON_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.5 +PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=13a03ea87aa67c7db414bf245fbcc623555c783a34d8ba1d7d701fd42717c366 +PAIMON_AWS_C_SDKUTILS_PKG_NAME=aws-c-sdkutils-${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz +PAIMON_AWS_S2N_BUILD_VERSION=v1.7.4 +PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM=af5ce0783fd9e05ed1899fda0c76e02fa5dd92128018e5bfe71634312d2ce8e7 +PAIMON_AWS_S2N_PKG_NAME=s2n-${PAIMON_AWS_S2N_BUILD_VERSION}.zip + PAIMON_FMT_BUILD_VERSION=11.2.0 PAIMON_FMT_BUILD_SHA256_CHECKSUM=bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af PAIMON_FMT_PKG_NAME=fmt-${PAIMON_FMT_BUILD_VERSION}.tar.gz @@ -84,8 +113,8 @@ PAIMON_RAPIDJSON_BUILD_VERSION=232389d4f1012dddec4ef84861face2d2ba85709 PAIMON_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc5984a19ed5ae75d090064806 PAIMON_RAPIDJSON_PKG_NAME=rapidjson-${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz -PAIMON_LUMINA_BUILD_VERSION=0.2.3 -PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=49de9548a8b81f6a5e3794e8e65fb32023a1c4bdb019028ce55cbcc98c12be97 +PAIMON_LUMINA_BUILD_VERSION=0.3.1 +PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=2aed1ae238c866c12ee01fad17d52651a9029cba1b972b23cd640959310e2149 PAIMON_LUMINA_PKG_NAME=lumina_release-${PAIMON_LUMINA_BUILD_VERSION}.tar.gz PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2 @@ -127,8 +156,17 @@ DEPENDENCIES=( "PAIMON_TBB_URL ${PAIMON_TBB_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/uxlfoundation/oneTBB/archive/refs/tags/${PAIMON_TBB_BUILD_VERSION}.tar.gz" "PAIMON_ORC_URL ${PAIMON_ORC_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/orc/archive/refs/tags/${PAIMON_ORC_BUILD_VERSION}.tar.gz" "PAIMON_GTEST_URL ${PAIMON_GTEST_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/googletest/archive/release-${PAIMON_GTEST_BUILD_VERSION}.tar.gz" + "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz" "PAIMON_ARROW_URL ${PAIMON_ARROW_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/arrow/releases/download/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz" "PAIMON_AVRO_URL ${PAIMON_AVRO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/avro/archive/${PAIMON_AVRO_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_AUTH_URL ${PAIMON_AWS_C_AUTH_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-auth/archive/refs/tags/${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_CAL_URL ${PAIMON_AWS_C_CAL_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-cal/archive/refs/tags/${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_COMMON_URL ${PAIMON_AWS_C_COMMON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-common/archive/refs/tags/${PAIMON_AWS_C_COMMON_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_COMPRESSION_URL ${PAIMON_AWS_C_COMPRESSION_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-compression/archive/refs/tags/${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_HTTP_URL ${PAIMON_AWS_C_HTTP_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-http/archive/refs/tags/${PAIMON_AWS_C_HTTP_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_IO_URL ${PAIMON_AWS_C_IO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-io/archive/refs/tags/${PAIMON_AWS_C_IO_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_C_SDKUTILS_URL ${PAIMON_AWS_C_SDKUTILS_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-sdkutils/archive/refs/tags/${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz" + "PAIMON_AWS_S2N_URL ${PAIMON_AWS_S2N_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip" "PAIMON_FMT_URL ${PAIMON_FMT_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/fmtlib/fmt/archive/refs/tags/${PAIMON_FMT_BUILD_VERSION}.tar.gz" "PAIMON_GLOG_URL ${PAIMON_GLOG_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/glog/archive/${PAIMON_GLOG_BUILD_VERSION}.tar.gz" "PAIMON_RAPIDJSON_URL ${PAIMON_RAPIDJSON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/miloyip/rapidjson/archive/${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz"