From 2848f598519de2410fe70834cd0f8f8503bbc97e Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Sun, 12 Jul 2026 02:25:14 +0800 Subject: [PATCH] [Opt](bitmap) Optimize bitmap op count rewrites --- be/src/core/value/bitmap_value.h | 4 +- be/src/exprs/function/function.h | 1 + be/src/exprs/function/function_bitmap.cpp | 101 +++--------- .../function/function_bitmap_variadic.cpp | 153 ++++++++++++++++-- .../exprs/function/simple_function_factory.h | 5 + be/src/exprs/vectorized_fn_call.cpp | 5 +- be/src/format_v2/expr/cast.cpp | 5 +- .../exprs/function/function_bitmap_test.cpp | 81 ++++++++-- .../function/simple_function_factory_test.cpp | 35 ++++ .../doris/catalog/BuiltinScalarFunctions.java | 10 +- .../load/NereidsStreamLoadPlanner.java | 1 + .../expression/ExpressionNormalization.java | 4 +- .../rules/expression/ExpressionRuleType.java | 3 +- .../rules/BitmapCountToBitmapOpCount.java | 67 ++++++++ .../rules/FoldConstantRuleOnBE.java | 1 + .../functions/scalar/BitmapAndCount.java | 8 +- .../functions/scalar/BitmapAndNot.java | 73 --------- .../functions/scalar/BitmapAndNotAlias.java | 73 --------- .../functions/scalar/BitmapAndNotCount.java | 8 +- .../scalar/BitmapAndNotCountAlias.java | 78 --------- .../functions/scalar/BitmapOrCount.java | 7 +- .../functions/scalar/BitmapXorCount.java | 8 +- .../visitor/ScalarFunctionVisitor.java | 15 -- .../apache/doris/qe/CoordinatorContext.java | 1 + .../apache/doris/qe/NereidsCoordinator.java | 1 + .../org/apache/doris/qe/SessionVariable.java | 1 + .../rules/analysis/FunctionRegistryTest.java | 21 +++ .../rules/BitmapCountToBitmapOpCountTest.java | 115 +++++++++++++ gensrc/thrift/PaloInternalService.thrift | 1 + .../bitmap_functions/test_bitmap_function.out | 18 +++ .../test_bitmap_function.groovy | 6 + 31 files changed, 531 insertions(+), 379 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h index 3b7b05d04a10c3..5a2e472a1ce0be 100644 --- a/be/src/core/value/bitmap_value.h +++ b/be/src/core/value/bitmap_value.h @@ -1796,8 +1796,8 @@ class BitmapValue { } case SET: { uint64_t cardinality = _set.size(); - for (auto v : _set) { - if (!rhs._set.contains(v)) { + for (auto v : rhs._set) { + if (!_set.contains(v)) { ++cardinality; } } diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 171e889f79b21d..7b8a9105d17779 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -56,6 +56,7 @@ namespace doris { struct FunctionAttr { bool new_version_unix_timestamp {false}; + bool new_version_bitmap_op_count {false}; }; #define RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(TYPE) \ diff --git a/be/src/exprs/function/function_bitmap.cpp b/be/src/exprs/function/function_bitmap.cpp index 35341f297640b0..e867add61adaa5 100644 --- a/be/src/exprs/function/function_bitmap.cpp +++ b/be/src/exprs/function/function_bitmap.cpp @@ -577,52 +577,6 @@ struct BitmapNot { } }; -struct NameBitmapAndNot { - static constexpr auto name = "bitmap_and_not"; -}; - -template -struct BitmapAndNot { - using ResultDataType = DataTypeBitMap; - using T0 = typename LeftDataType::FieldType; - using T1 = typename RightDataType::FieldType; - using TData = std::vector; - - static void vector_vector(const TData& lvec, const TData& rvec, TData& res) { - size_t size = lvec.size(); - BitmapValue mid_data; - for (size_t i = 0; i < size; ++i) { - mid_data = lvec[i]; - mid_data &= rvec[i]; - res[i] = lvec[i]; - res[i] -= mid_data; - mid_data.reset(); - } - } - static void vector_scalar(const TData& lvec, const BitmapValue& rval, TData& res) { - size_t size = lvec.size(); - BitmapValue mid_data; - for (size_t i = 0; i < size; ++i) { - mid_data = lvec[i]; - mid_data &= rval; - res[i] = lvec[i]; - res[i] -= mid_data; - mid_data.reset(); - } - } - static void scalar_vector(const BitmapValue& lval, const TData& rvec, TData& res) { - size_t size = rvec.size(); - BitmapValue mid_data; - for (size_t i = 0; i < size; ++i) { - mid_data = lval; - mid_data &= rvec[i]; - res[i] = lval; - res[i] -= mid_data; - mid_data.reset(); - } - } -}; - struct NameBitmapAndNotCount { static constexpr auto name = "bitmap_and_not_count"; }; @@ -637,32 +591,20 @@ struct BitmapAndNotCount { static void vector_vector(const TData& lvec, const TData& rvec, ResTData* res) { size_t size = lvec.size(); - BitmapValue mid_data; for (size_t i = 0; i < size; ++i) { - mid_data = lvec[i]; - mid_data &= rvec[i]; - res[i] = lvec[i].andnot_cardinality(mid_data); - mid_data.reset(); + res[i] = lvec[i].andnot_cardinality(rvec[i]); } } static void scalar_vector(const BitmapValue& lval, const TData& rvec, ResTData* res) { size_t size = rvec.size(); - BitmapValue mid_data; for (size_t i = 0; i < size; ++i) { - mid_data = lval; - mid_data &= rvec[i]; - res[i] = lval.andnot_cardinality(mid_data); - mid_data.reset(); + res[i] = lval.andnot_cardinality(rvec[i]); } } static void vector_scalar(const TData& lvec, const BitmapValue& rval, ResTData* res) { size_t size = lvec.size(); - BitmapValue mid_data; for (size_t i = 0; i < size; ++i) { - mid_data = lvec[i]; - mid_data &= rval; - res[i] = lvec[i].andnot_cardinality(mid_data); - mid_data.reset(); + res[i] = lvec[i].andnot_cardinality(rval); } } }; @@ -682,8 +624,10 @@ ColumnPtr handle_bitmap_op_count_null_value(ColumnPtr& src, const Block& block, const ColumnNumbers& args, uint32_t result, size_t input_rows_count) { MutableColumnPtr mutable_src = IColumn::mutate(std::move(src)); - auto* nullable = assert_cast(mutable_src.get()); - auto* src_not_nullable_mutable = &nullable->get_nested_column(); + auto* src_not_nullable_mutable = mutable_src.get(); + if (auto* nullable = check_and_get_column(*mutable_src)) { + src_not_nullable_mutable = &nullable->get_nested_column(); + } auto* __restrict count_data = assert_cast(src_not_nullable_mutable)->get_data().data(); @@ -734,28 +678,32 @@ Status execute_bitmap_op_count_null_to_zero( return Status::OK(); } -template +template class FunctionBitmapAndNotCount : public IFunction { public: using LeftDataType = DataTypeBitMap; using RightDataType = DataTypeBitMap; using ResultDataType = typename BitmapAndNotCount::ResultDataType; - static constexpr auto name = FunctionName::name; + static constexpr auto name = NewVersion ? "bitmap_and_not_count_v2" : FunctionName::name; static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 2; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - bool return_nullable = false; - // result is nullable only when any columns is nullable for bitmap_and_not_count - for (size_t i = 0; i < arguments.size(); ++i) { - if (arguments[i]->is_nullable()) { - return_nullable = true; - break; + if constexpr (NewVersion) { + return std::make_shared(); + } else { + bool return_nullable = false; + // result is nullable only when any columns is nullable for bitmap_and_not_count + for (size_t i = 0; i < arguments.size(); ++i) { + if (arguments[i]->is_nullable()) { + return_nullable = true; + break; + } } + auto result_type = std::make_shared(); + return return_nullable ? make_nullable(result_type) : result_type; } - auto result_type = std::make_shared(); - return return_nullable ? make_nullable(result_type) : result_type; } bool use_default_implementation_for_nulls() const override { @@ -1264,8 +1212,6 @@ using FunctionBitmapToBase64 = FunctionUnaryToType; using FunctionBitmapNot = FunctionBinaryToType; -using FunctionBitmapAndNot = - FunctionBinaryToType; using FunctionBitmapContains = FunctionBinaryToType; using FunctionBitmapRemove = @@ -1294,9 +1240,10 @@ void register_function_bitmap(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_alias(NameBitmapAndNot::name, "bitmap_andnot"); + factory.register_alias(NameBitmapNot::name, "bitmap_and_not"); + factory.register_alias(NameBitmapNot::name, "bitmap_andnot"); factory.register_function>(); + factory.register_function>(); factory.register_alias(NameBitmapAndNotCount::name, "bitmap_andnot_count"); factory.register_function(); factory.register_function(); diff --git a/be/src/exprs/function/function_bitmap_variadic.cpp b/be/src/exprs/function/function_bitmap_variadic.cpp index 11cfcb9ef81e25..b290fdb1f33c5d 100644 --- a/be/src/exprs/function/function_bitmap_variadic.cpp +++ b/be/src/exprs/function/function_bitmap_variadic.cpp @@ -162,12 +162,30 @@ Status execute_bitmap_op_count_null_to_zero( const std::function& exec_impl_func); -template +template class FunctionBitMapVariadic : public IFunction { + static_assert(!NewVersion || (NewVersion && (std::is_same_v || + std::is_same_v || + std::is_same_v))); + public: - static constexpr auto name = Impl::name; + static constexpr auto name = []() constexpr { + if constexpr (!NewVersion) { + return Impl::name; + } else if constexpr (std::is_same_v) { + return "bitmap_or_count_v2"; + } else if constexpr (std::is_same_v) { + return "bitmap_and_count_v2"; + } else if constexpr (std::is_same_v) { + return "bitmap_xor_count_v2"; + } else { + return Impl::name; + } + }(); - static FunctionPtr create() { return std::make_shared(); } + static FunctionPtr create() { + return std::make_shared>(); + } String get_name() const override { return name; } @@ -177,6 +195,9 @@ class FunctionBitMapVariadic : public IFunction { DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { using ResultDataType = typename Impl::ResultDataType; + if constexpr (NewVersion) { + return std::make_shared(); + } if (std::is_same_v || is_count()) { bool return_nullable = false; // result is nullable only when any columns is nullable for bitmap_or and bitmap_or_count @@ -202,7 +223,8 @@ class FunctionBitMapVariadic : public IFunction { Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const override { - if (std::is_same_v || std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v) { auto impl_func = [&](FunctionContext* context, Block& block, const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) { @@ -219,12 +241,6 @@ class FunctionBitMapVariadic : public IFunction { const ColumnNumbers& arguments, uint32_t result, size_t input_rows_count) const { size_t argument_size = arguments.size(); - std::vector argument_columns(argument_size); - - for (size_t i = 0; i < argument_size; ++i) { - argument_columns[i] = - block.get_by_position(arguments[i]).column->convert_to_full_column_if_const(); - } using ResultDataType = typename Impl::ResultDataType; //DataTypeBitMap or DataTypeInt64 using ColVecResult = std::conditional_t, @@ -244,8 +260,26 @@ class FunctionBitMapVariadic : public IFunction { auto& vec_res = col_res->get_data(); vec_res.resize(input_rows_count); - RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(), argument_size, - input_rows_count, vec_res, col_res_nulls.get())); + bool used_binary_count_path = false; + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (argument_size == 2) { + RETURN_IF_ERROR(execute_binary_bitmap_count( + block.get_by_position(arguments[0]).column, + block.get_by_position(arguments[1]).column, input_rows_count, vec_res)); + used_binary_count_path = true; + } + } + if (!used_binary_count_path) { + std::vector argument_columns(argument_size); + for (size_t i = 0; i < argument_size; ++i) { + argument_columns[i] = block.get_by_position(arguments[i]) + .column->convert_to_full_column_if_const(); + } + RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(), argument_size, + input_rows_count, vec_res, col_res_nulls.get())); + } + if (!use_default_implementation_for_nulls() && result_info.type->is_nullable()) { block.replace_by_position( result, ColumnNullable::create(std::move(col_res), std::move(col_res_nulls))); @@ -260,6 +294,93 @@ class FunctionBitMapVariadic : public IFunction { return (std::is_same_v || std::is_same_v || std::is_same_v); } + + template + Status execute_binary_bitmap_count(const ColumnPtr& lhs_column, const ColumnPtr& rhs_column, + size_t input_rows_count, ColumnInt64::Container& res) const { + static_assert(std::is_same_v || + std::is_same_v || + std::is_same_v); + + struct BitmapColumnAccessor { + const std::vector* values = nullptr; + const BitmapValue* const_value = nullptr; + const ColumnUInt8::value_type* null_map_data = nullptr; + bool is_const = false; + bool is_const_null = false; + + bool is_null_at(size_t row) const { + return is_const ? is_const_null : (null_map_data && null_map_data[row]); + } + + const BitmapValue& get_value(size_t row) const { + return is_const ? *const_value : (*values)[row]; + } + }; + + auto make_accessor = [](const ColumnPtr& column) { + BitmapColumnAccessor accessor; + const auto& [data_column_ptr, is_const] = unpack_if_const(column); + accessor.is_const = is_const; + const IColumn* data_column = data_column_ptr.get(); + + if (const auto* nullable = check_and_get_column(*data_column)) { + if (accessor.is_const) { + accessor.is_const_null = nullable->is_null_at(0); + } else { + accessor.null_map_data = nullable->get_null_map_data().data(); + } + data_column = nullable->get_nested_column_ptr().get(); + } + + const auto* bitmap_column = assert_cast(data_column); + if (accessor.is_const) { + accessor.const_value = &bitmap_column->get_data()[0]; + } else { + accessor.values = &bitmap_column->get_data(); + } + + return accessor; + }; + + const auto lhs = make_accessor(lhs_column); + const auto rhs = make_accessor(rhs_column); + + for (size_t row = 0; row < input_rows_count; ++row) { + const bool lhs_is_null = lhs.is_null_at(row); + const bool rhs_is_null = rhs.is_null_at(row); + + if (lhs_is_null && rhs_is_null) { + res[row] = 0; + continue; + } + + if constexpr (std::is_same_v) { + if (lhs_is_null) { + res[row] = rhs.get_value(row).cardinality(); + } else if (rhs_is_null) { + res[row] = lhs.get_value(row).cardinality(); + } else { + res[row] = lhs.get_value(row).or_cardinality(rhs.get_value(row)); + } + } else if constexpr (std::is_same_v) { + res[row] = (lhs_is_null || rhs_is_null) + ? 0 + : lhs.get_value(row).and_cardinality(rhs.get_value(row)); + } else { + if (lhs_is_null || rhs_is_null) { + res[row] = 0; + } else { + const auto& lhs_value = lhs.get_value(row); + const auto& rhs_value = rhs.get_value(row); + uint64_t inter = lhs_value.and_cardinality(rhs_value); + res[row] = lhs_value.cardinality() + rhs_value.cardinality() - 2 * inter; + } + } + } + + return Status::OK(); + } }; using FunctionBitmapOr = FunctionBitMapVariadic; @@ -268,6 +389,9 @@ using FunctionBitmapAnd = FunctionBitMapVariadic; using FunctionBitmapOrCount = FunctionBitMapVariadic; using FunctionBitmapAndCount = FunctionBitMapVariadic; using FunctionBitmapXorCount = FunctionBitMapVariadic; +using FunctionBitmapOrCountV2 = FunctionBitMapVariadic; +using FunctionBitmapAndCountV2 = FunctionBitMapVariadic; +using FunctionBitmapXorCountV2 = FunctionBitMapVariadic; void register_function_bitmap_variadic(SimpleFunctionFactory& factory) { factory.register_function(); @@ -276,5 +400,8 @@ void register_function_bitmap_variadic(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/function/simple_function_factory.h b/be/src/exprs/function/simple_function_factory.h index c0c3fc1ad60365..1b7ac1d8c708da 100644 --- a/be/src/exprs/function/simple_function_factory.h +++ b/be/src/exprs/function/simple_function_factory.h @@ -204,6 +204,11 @@ class SimpleFunctionFactory { attr.new_version_unix_timestamp) { key_str += "_new"; } + if (attr.new_version_bitmap_op_count && + (key_str == "bitmap_and_count" || key_str == "bitmap_or_count" || + key_str == "bitmap_xor_count" || key_str == "bitmap_and_not_count")) { + key_str += "_v2"; + } temporary_function_update(be_version, key_str); diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 8cc189f8b9939e..101791ab7293bc 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -164,7 +164,10 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, // get the function. won't prepare function. _function = SimpleFunctionFactory::instance().get_function( _fn.name.function_name, argument_template, _data_type, - {.new_version_unix_timestamp = state->query_options().new_version_unix_timestamp}, + {.new_version_unix_timestamp = state->query_options().new_version_unix_timestamp, + .new_version_bitmap_op_count = + state->query_options().__isset.new_version_bitmap_op_count && + state->query_options().new_version_bitmap_op_count}, state->be_exec_version()); } if (_function == nullptr) { diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp index efeb9d851deb22..c1435732a9e136 100644 --- a/be/src/format_v2/expr/cast.cpp +++ b/be/src/format_v2/expr/cast.cpp @@ -56,7 +56,10 @@ Status Cast::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContex // get the function. won't prepare function. _function = SimpleFunctionFactory::instance().get_function( "CAST", argument_template, _data_type, - {.new_version_unix_timestamp = state->query_options().new_version_unix_timestamp}, + {.new_version_unix_timestamp = state->query_options().new_version_unix_timestamp, + .new_version_bitmap_op_count = + state->query_options().__isset.new_version_bitmap_op_count && + state->query_options().new_version_bitmap_op_count}, state->be_exec_version()); if (_function == nullptr) { return Status::InternalError("Could not find function {} ", _expr_name); diff --git a/be/test/exprs/function/function_bitmap_test.cpp b/be/test/exprs/function/function_bitmap_test.cpp index 6e18e10d2e354f..b4ad89b3d3651f 100644 --- a/be/test/exprs/function/function_bitmap_test.cpp +++ b/be/test/exprs/function/function_bitmap_test.cpp @@ -323,9 +323,9 @@ TEST(function_bitmap_test, function_bitmap_and_count) { BitmapValue bitmap1({1, 2, 3}); BitmapValue bitmap2({3, 4, 5}); BitmapValue empty_bitmap; - DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)0}, - {{&bitmap1, &bitmap1}, (int64_t)3}, - {{&bitmap1, &bitmap2}, (int64_t)1}}; + DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)0}, {{&bitmap1, &bitmap1}, (int64_t)3}, + {{&bitmap1, &bitmap2}, (int64_t)1}, {{Null(), &bitmap1}, (int64_t)0}, + {{&bitmap1, Null()}, (int64_t)0}, {{Null(), Null()}, (int64_t)0}}; static_cast(check_function(func_name, input_types, data_set)); @@ -354,12 +354,26 @@ TEST(function_bitmap_test, function_bitmap_or_count) { BitmapValue bitmap2({1, 2, 3, 4}); BitmapValue bitmap3({2, 3}); BitmapValue empty_bitmap; - DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, - {{&bitmap2, &bitmap3}, (int64_t)4}, - {{&bitmap1, &bitmap3}, (int64_t)3}}; + DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, {{&bitmap2, &bitmap3}, (int64_t)4}, + {{&bitmap1, &bitmap3}, (int64_t)3}, {{Null(), &bitmap1}, (int64_t)3}, + {{&bitmap1, Null()}, (int64_t)3}, {{Null(), Null()}, (int64_t)0}}; static_cast(check_function(func_name, input_types, data_set)); + { + const auto old_config = config::enable_set_in_bitmap_value; + config::enable_set_in_bitmap_value = true; + BitmapValue short_bitmap1({1, 2}); + BitmapValue short_bitmap2({2, 3, 4, 5}); + EXPECT_EQ(short_bitmap1.get_type_code(), BitmapTypeCode::SET); + EXPECT_EQ(short_bitmap2.get_type_code(), BitmapTypeCode::SET); + + DataSet set_data_set = {{{&short_bitmap1, &short_bitmap2}, (int64_t)5}}; + static_cast( + check_function(func_name, input_types, set_data_set)); + config::enable_set_in_bitmap_value = old_config; + } + { InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP, PrimitiveType::TYPE_BITMAP, PrimitiveType::TYPE_BITMAP}; @@ -388,9 +402,9 @@ TEST(function_bitmap_test, function_bitmap_xor_count) { BitmapValue bitmap3({2, 3}); BitmapValue bitmap4({1, 2, 6}); BitmapValue empty_bitmap; - DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, - {{&bitmap2, &bitmap3}, (int64_t)2}, - {{&bitmap1, &bitmap4}, (int64_t)2}}; + DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, {{&bitmap2, &bitmap3}, (int64_t)2}, + {{&bitmap1, &bitmap4}, (int64_t)2}, {{Null(), &bitmap1}, (int64_t)0}, + {{&bitmap1, Null()}, (int64_t)0}, {{Null(), Null()}, (int64_t)0}}; static_cast(check_function(func_name, input_types, data_set)); @@ -420,13 +434,54 @@ TEST(function_bitmap_test, function_bitmap_and_not_count) { BitmapValue bitmap3({33, 5, std::numeric_limits::max()}); BitmapValue empty_bitmap; - DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, //1,2,3 - {{&bitmap2, Null()}, (int64_t)0}, - {{&bitmap2, &bitmap3}, (int64_t)3}, //0,3,4 - {{&bitmap1, &bitmap2}, (int64_t)2}}; //1,2 + DataSet data_set = { + {{&bitmap1, &empty_bitmap}, (int64_t)3}, //1,2,3 + {{Null(), &bitmap1}, (int64_t)0}, {{&bitmap2, Null()}, (int64_t)0}, + {{Null(), Null()}, (int64_t)0}, {{&bitmap2, &bitmap3}, (int64_t)3}, //0,3,4 + {{&bitmap1, &bitmap2}, (int64_t)2}}; //1,2 static_cast(check_function(func_name, input_types, data_set)); } + +TEST(function_bitmap_test, function_bitmap_op_count_v2_nullable_args) { + InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP, PrimitiveType::TYPE_BITMAP}; + BitmapValue bitmap1({1, 2, 3}); + BitmapValue bitmap2({3, 4, 5}); + + { + DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)1}, + {{Null(), &bitmap1}, (int64_t)0}, + {{&bitmap1, Null()}, (int64_t)0}, + {{Null(), Null()}, (int64_t)0}}; + static_cast( + check_function("bitmap_and_count_v2", input_types, data_set)); + } + { + DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)5}, + {{Null(), &bitmap1}, (int64_t)3}, + {{&bitmap1, Null()}, (int64_t)3}, + {{Null(), Null()}, (int64_t)0}}; + static_cast( + check_function("bitmap_or_count_v2", input_types, data_set)); + } + { + DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)4}, + {{Null(), &bitmap1}, (int64_t)0}, + {{&bitmap1, Null()}, (int64_t)0}, + {{Null(), Null()}, (int64_t)0}}; + static_cast( + check_function("bitmap_xor_count_v2", input_types, data_set)); + } + { + DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)2}, + {{Null(), &bitmap1}, (int64_t)0}, + {{&bitmap1, Null()}, (int64_t)0}, + {{Null(), Null()}, (int64_t)0}}; + static_cast(check_function("bitmap_and_not_count_v2", + input_types, data_set)); + } +} + TEST(function_bitmap_test, function_bitmap_and_not_count_alias) { std::string func_name = "bitmap_andnot_count"; InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP, PrimitiveType::TYPE_BITMAP}; diff --git a/be/test/exprs/function/simple_function_factory_test.cpp b/be/test/exprs/function/simple_function_factory_test.cpp index 86f344a3f03db7..dc8dcdf858343b 100644 --- a/be/test/exprs/function/simple_function_factory_test.cpp +++ b/be/test/exprs/function/simple_function_factory_test.cpp @@ -20,7 +20,12 @@ #include #include +#include +#include +#include +#include "core/data_type/data_type_bitmap.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" namespace doris { @@ -95,4 +100,34 @@ TEST_F(SimpleFunctionFactoryTest, test_return_all) { } } +TEST_F(SimpleFunctionFactoryTest, test_bitmap_count_new_version_return_type) { + ColumnsWithTypeAndName arguments = { + {nullptr, make_nullable(std::make_shared()), ""}, + {nullptr, make_nullable(std::make_shared()), ""}}; + auto& factory = SimpleFunctionFactory::instance(); + + for (const auto& [function_name, new_function_name] : + std::vector> { + {"bitmap_and_count", "bitmap_and_count_v2"}, + {"bitmap_or_count", "bitmap_or_count_v2"}, + {"bitmap_xor_count", "bitmap_xor_count_v2"}, + {"bitmap_and_not_count", "bitmap_and_not_count_v2"}}) { + auto old_builder = std::dynamic_pointer_cast( + factory.function_creators.find(function_name)->second()); + ASSERT_NE(old_builder, nullptr); + EXPECT_TRUE(old_builder->get_return_type(arguments)->is_nullable()); + + auto new_builder = std::dynamic_pointer_cast( + factory.function_creators.find(new_function_name)->second()); + ASSERT_NE(new_builder, nullptr); + EXPECT_FALSE(new_builder->get_return_type(arguments)->is_nullable()); + + auto new_func = SimpleFunctionFactory::instance().get_function( + function_name, arguments, std::make_shared(), + {.new_version_bitmap_op_count = true}, BeExecVersionManager::get_newest_version()); + EXPECT_EQ(new_func->get_name(), new_function_name); + EXPECT_FALSE(new_func->get_return_type()->is_nullable()); + } +} + } // namespace doris \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index aa43f0609823dd..54c393ee0dac53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -103,10 +103,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitTest; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNot; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotAlias; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCountAlias; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; @@ -677,10 +674,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(BitLength.class, "bit_length"), scalar(BitmapAnd.class, "bitmap_and"), scalar(BitmapAndCount.class, "bitmap_and_count"), - scalar(BitmapAndNot.class, "bitmap_and_not"), - scalar(BitmapAndNotAlias.class, "bitmap_andnot"), - scalar(BitmapAndNotCount.class, "bitmap_and_not_count"), - scalar(BitmapAndNotCountAlias.class, "bitmap_andnot_count"), + scalar(BitmapAndNotCount.class, "bitmap_and_not_count", "bitmap_andnot_count"), scalar(BitmapContains.class, "bitmap_contains"), scalar(BitmapCount.class, "bitmap_count"), scalar(BitmapEmpty.class, "bitmap_empty"), @@ -693,7 +687,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(BitmapHash64.class, "bitmap_hash64"), scalar(BitmapMax.class, "bitmap_max"), scalar(BitmapMin.class, "bitmap_min"), - scalar(BitmapNot.class, "bitmap_not"), + scalar(BitmapNot.class, "bitmap_not", "bitmap_and_not", "bitmap_andnot"), scalar(BitmapOr.class, "bitmap_or"), scalar(BitmapOrCount.class, "bitmap_or_count"), scalar(BitmapRemove.class, "bitmap_remove"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 69d68365189af9..9085266228ee9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -327,6 +327,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde queryOptions.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); queryOptions.setNewVersionUnixTimestamp(true); queryOptions.setNewVersionPercentile(true); + queryOptions.setNewVersionBitmapOpCount(true); params.setQueryOptions(queryOptions); TQueryGlobals queryGlobals = new TQueryGlobals(); queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java index c6b129e8c5088b..6155734100e15d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.expression; import org.apache.doris.nereids.rules.expression.check.CheckCast; +import org.apache.doris.nereids.rules.expression.rules.BitmapCountToBitmapOpCount; import org.apache.doris.nereids.rules.expression.rules.ConcatWsMultiArrayToOne; import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast; import org.apache.doris.nereids.rules.expression.rules.DigitalMaskingConvert; @@ -76,7 +77,8 @@ public class ExpressionNormalization extends ExpressionRewrite { MergeDateTrunc.INSTANCE, NormalizeElementAt.INSTANCE, CheckCast.INSTANCE, - SimplifyEqualBooleanLiteral.INSTANCE + SimplifyEqualBooleanLiteral.INSTANCE, + BitmapCountToBitmapOpCount.INSTANCE ) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java index b2e44c81c1620a..a3409004ca7720 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java @@ -69,7 +69,8 @@ public enum ExpressionRuleType { TIMESTAMP_TO_ADD_TIME, TOPN_TO_MAX, REWRITE_DEFAULT_EXPRESSION, - ADD_SESSION_VAR_GUARD; + ADD_SESSION_VAR_GUARD, + BITMAP_COUNT_TO_BITMAP_OP_COUNT; public int type() { return ordinal(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java new file mode 100644 index 00000000000000..8292d5cf62de1a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; +import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.rules.expression.ExpressionRuleType; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** Rewrite bitmap_count(bitmap_op(...)) to corresponding bitmap_op_count function. */ +public class BitmapCountToBitmapOpCount implements ExpressionPatternRuleFactory { + public static final BitmapCountToBitmapOpCount INSTANCE = new BitmapCountToBitmapOpCount(); + + @Override + public List> buildRules() { + return ImmutableList.of( + matchesType(BitmapCount.class).then(BitmapCountToBitmapOpCount::rewrite) + .toRule(ExpressionRuleType.BITMAP_COUNT_TO_BITMAP_OP_COUNT) + ); + } + + private static Expression rewrite(BitmapCount bitmapCount) { + Expression child = bitmapCount.child(0); + if (child instanceof BitmapAnd) { + return new BitmapAndCount(child.child(0), child.child(1), varArgs(child)); + } else if (child instanceof BitmapOr) { + return new BitmapOrCount(child.child(0), child.child(1), varArgs(child)); + } else if (child instanceof BitmapXor) { + return new BitmapXorCount(child.child(0), child.child(1), varArgs(child)); + } else if (child instanceof BitmapNot) { + return new BitmapAndNotCount(child.child(0), child.child(1)); + } + return bitmapCount; + } + + private static Expression[] varArgs(Expression expression) { + return expression.children().subList(2, expression.arity()).toArray(new Expression[0]); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 50eee509628911..3f6812405e864b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -322,6 +322,7 @@ private static Map evalOnBE(Map> tQueryOptions.setEnableDecimal256(context.getSessionVariable().isEnableDecimal256()); tQueryOptions.setNewVersionUnixTimestamp(true); tQueryOptions.setNewVersionPercentile(true); + tQueryOptions.setNewVersionBitmapOpCount(true); tQueryOptions.setEnableStrictCast(SessionVariable.enableStrictCast()); TFoldConstantParams tParams = new TFoldConstantParams(paramMap, queryGlobals); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java index 6266b02735fa01..70a62d3a41ada4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -34,7 +35,7 @@ * ScalarFunction 'bitmap_and_count'. This class is generated by GenerateFunction. */ public class BitmapAndCount extends ScalarFunction - implements ExplicitlyCastableSignature { + implements ExplicitlyCastableSignature, AlwaysNotNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE, BitmapType.INSTANCE) @@ -70,9 +71,4 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBitmapAndCount(this, context); } - - @Override - public boolean nullable() { - return children().stream().anyMatch(Expression::nullable); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java deleted file mode 100644 index 835b141027d053..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BitmapType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'bitmap_and_not'. This class is generated by GenerateFunction. - */ -public class BitmapAndNot extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE, BitmapType.INSTANCE) - ); - - /** - * constructor with 2 arguments. - */ - public BitmapAndNot(Expression arg0, Expression arg1) { - super("bitmap_and_not", arg0, arg1); - } - - /** constructor for withChildren and reuse signature */ - private BitmapAndNot(ScalarFunctionParams functionParams) { - super(functionParams); - } - - /** - * withChildren. - */ - @Override - public BitmapAndNot withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); - return new BitmapAndNot(getFunctionParams(children)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitBitmapAndNot(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java deleted file mode 100644 index 1604590418409f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BitmapType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'bitmap_and_not'. This class is generated by GenerateFunction. - */ -public class BitmapAndNotAlias extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature, PropagateNullable { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE, BitmapType.INSTANCE) - ); - - /** - * constructor with 2 arguments. - */ - public BitmapAndNotAlias(Expression arg0, Expression arg1) { - super("bitmap_andnot", arg0, arg1); - } - - /** constructor for withChildren and reuse signature */ - private BitmapAndNotAlias(ScalarFunctionParams functionParams) { - super(functionParams); - } - - /** - * withChildren. - */ - @Override - public BitmapAndNotAlias withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); - return new BitmapAndNotAlias(getFunctionParams(children)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitBitmapAndNotAlias(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java index 0976bc6fe05d85..ac00f957a18473 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -34,7 +35,7 @@ * ScalarFunction 'bitmap_and_not_count'. This class is generated by GenerateFunction. */ public class BitmapAndNotCount extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature { + implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE, BitmapType.INSTANCE) @@ -70,9 +71,4 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBitmapAndNotCount(this, context); } - - @Override - public boolean nullable() { - return children().stream().anyMatch(Expression::nullable); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java deleted file mode 100644 index 36205c847e265c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.nereids.types.BitmapType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * ScalarFunction 'bitmap_and_not_count'. This class is generated by GenerateFunction. - */ -public class BitmapAndNotCountAlias extends ScalarFunction - implements BinaryExpression, ExplicitlyCastableSignature { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE, BitmapType.INSTANCE) - ); - - /** - * constructor with 2 arguments. - */ - public BitmapAndNotCountAlias(Expression arg0, Expression arg1) { - super("bitmap_andnot_count", arg0, arg1); - } - - /** constructor for withChildren and reuse signature */ - private BitmapAndNotCountAlias(ScalarFunctionParams functionParams) { - super(functionParams); - } - - /** - * withChildren. - */ - @Override - public BitmapAndNotCountAlias withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); - return new BitmapAndNotCountAlias(getFunctionParams(children)); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitBitmapAndNotCountAlias(this, context); - } - - @Override - public boolean nullable() { - return children().stream().anyMatch(Expression::nullable); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java index 92c51db51ac8f5..5165325d03f269 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -34,7 +35,7 @@ * ScalarFunction 'bitmap_or_count'. This class is generated by GenerateFunction. */ public class BitmapOrCount extends ScalarFunction - implements ExplicitlyCastableSignature { + implements ExplicitlyCastableSignature, AlwaysNotNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE, BitmapType.INSTANCE) @@ -70,8 +71,4 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBitmapOrCount(this, context); } - - public boolean nullable() { - return children().stream().anyMatch(Expression::nullable); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java index ff6742ebafc6c0..a532aa87e9f8ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -34,7 +35,7 @@ * ScalarFunction 'bitmap_xor_count'. This class is generated by GenerateFunction. */ public class BitmapXorCount extends ScalarFunction - implements ExplicitlyCastableSignature { + implements ExplicitlyCastableSignature, AlwaysNotNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE, BitmapType.INSTANCE) @@ -70,9 +71,4 @@ public List getSignatures() { public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBitmapXorCount(this, context); } - - @Override - public boolean nullable() { - return children().stream().anyMatch(Expression::nullable); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index ed6a8bcf9390af..77000169eb3072 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -113,10 +113,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitTest; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNot; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotAlias; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount; -import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCountAlias; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty; @@ -928,22 +925,10 @@ default R visitBitmapAndCount(BitmapAndCount bitmapAndCount, C context) { return visitScalarFunction(bitmapAndCount, context); } - default R visitBitmapAndNot(BitmapAndNot bitmapAndNot, C context) { - return visitScalarFunction(bitmapAndNot, context); - } - default R visitBitmapAndNotCount(BitmapAndNotCount bitmapAndNotCount, C context) { return visitScalarFunction(bitmapAndNotCount, context); } - default R visitBitmapAndNotAlias(BitmapAndNotAlias bitmapAndNotAlias, C context) { - return visitScalarFunction(bitmapAndNotAlias, context); - } - - default R visitBitmapAndNotCountAlias(BitmapAndNotCountAlias bitmapAndNotCountAlias, C context) { - return visitScalarFunction(bitmapAndNotCountAlias, context); - } - default R visitBitmapContains(BitmapContains bitmapContains, C context) { return visitScalarFunction(bitmapContains, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java index 1caae547f06595..f619d7f4917cc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -310,6 +310,7 @@ public static CoordinatorContext buildForLoad( queryOptions.setBeExecVersion(Config.be_exec_version); queryOptions.setNewVersionUnixTimestamp(true); queryOptions.setNewVersionPercentile(true); + queryOptions.setNewVersionBitmapOpCount(true); TQueryGlobals queryGlobals = new TQueryGlobals(); queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4a75a3cdb99090..7f9f4acffbed09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -506,6 +506,7 @@ private void setForInsert(long jobId) { this.coordinatorContext.queryOptions.setDisableFileCache(true); this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true); this.coordinatorContext.queryOptions.setNewVersionPercentile(true); + this.coordinatorContext.queryOptions.setNewVersionBitmapOpCount(true); } private void setForQuery() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2f92f9fc7b4888..598cc06f8a07e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5751,6 +5751,7 @@ public TQueryOptions toThrift() { tResult.setEnableInsertStrict(enableInsertStrict); tResult.setNewVersionUnixTimestamp(true); // once FE upgraded, always use new version tResult.setNewVersionPercentile(true); + tResult.setNewVersionBitmapOpCount(true); tResult.setHnswEfSearch(hnswEFSearch); tResult.setHnswCheckRelativeDistance(hnswCheckRelativeDistance); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java index 0b3e4cb19f63ff..df7f4ca319eea4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java @@ -21,16 +21,19 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.BuiltinFunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; @@ -87,6 +90,24 @@ public void testMultiName() { ); } + @Test + public void testBitmapAndNotCountAliasNotNullable() { + FunctionRegistry functionRegistry = new FunctionRegistry(); + ImmutableList arguments = ImmutableList.of( + new SlotReference("nullableBitmap1", BitmapType.INSTANCE, true), + new SlotReference("nullableBitmap2", BitmapType.INSTANCE, true)); + + FunctionBuilder aliasBuilder = functionRegistry.findFunctionBuilder("bitmap_andnot_count", arguments); + Expression aliasFunction = aliasBuilder.build("bitmap_andnot_count", arguments).first; + Assertions.assertInstanceOf(BitmapAndNotCount.class, aliasFunction); + Assertions.assertFalse(aliasFunction.nullable()); + + FunctionBuilder canonicalBuilder = functionRegistry.findFunctionBuilder("bitmap_and_not_count", arguments); + Expression canonicalFunction = canonicalBuilder.build("bitmap_and_not_count", arguments).first; + Assertions.assertInstanceOf(BitmapAndNotCount.class, canonicalFunction); + Assertions.assertFalse(canonicalFunction.nullable()); + } + @Test public void testOverrideArity() { // the substring function has 2 override functions: diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java new file mode 100644 index 00000000000000..7275e9eedb7be3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor; +import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount; +import org.apache.doris.nereids.types.BitmapType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BitmapCountToBitmapOpCountTest extends ExpressionRewriteTestHelper { + + private final SlotReference bitmap1 = new SlotReference("bitmap1", BitmapType.INSTANCE, true); + private final SlotReference bitmap2 = new SlotReference("bitmap2", BitmapType.INSTANCE, true); + private final SlotReference bitmap3 = new SlotReference("bitmap3", BitmapType.INSTANCE, true); + + @BeforeEach + void setUp() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(BitmapCountToBitmapOpCount.INSTANCE) + )); + } + + @Test + void testRewriteBitmapAnd() { + assertRewrite( + new BitmapCount(new BitmapAnd(bitmap1, bitmap2)), + new BitmapAndCount(bitmap1, bitmap2) + ); + } + + @Test + void testRewriteBitmapAndWithVarArgs() { + assertRewrite( + new BitmapCount(new BitmapAnd(bitmap1, bitmap2, bitmap3)), + new BitmapAndCount(bitmap1, bitmap2, bitmap3) + ); + } + + @Test + void testRewriteBitmapOr() { + assertRewrite( + new BitmapCount(new BitmapOr(bitmap1, bitmap2, bitmap3)), + new BitmapOrCount(bitmap1, bitmap2, bitmap3) + ); + } + + @Test + void testRewriteBitmapXor() { + assertRewrite( + new BitmapCount(new BitmapXor(bitmap1, bitmap2)), + new BitmapXorCount(bitmap1, bitmap2) + ); + } + + @Test + void testRewriteBitmapNot() { + assertRewrite( + new BitmapCount(new BitmapNot(bitmap1, bitmap2)), + new BitmapAndNotCount(bitmap1, bitmap2) + ); + } + + @Test + void testKeepPlainBitmapCount() { + assertRewrite( + new BitmapCount(bitmap1), + new BitmapCount(bitmap1) + ); + } + + @Test + void testBitmapOpCountNotNullableMetadata() { + Assertions.assertFalse(new BitmapAndCount(bitmap1, bitmap2).nullable()); + Assertions.assertFalse(new BitmapOrCount(bitmap1, bitmap2).nullable()); + Assertions.assertFalse(new BitmapXorCount(bitmap1, bitmap2).nullable()); + Assertions.assertFalse(new BitmapAndNotCount(bitmap1, bitmap2).nullable()); + } + + @Test + void testRewriteToNotNullableBitmapOpCount() { + Expression rewrittenExpression = executor.rewrite(new BitmapCount(new BitmapAnd(bitmap1, bitmap2)), context); + Assertions.assertInstanceOf(BitmapAndCount.class, rewrittenExpression); + Assertions.assertFalse(rewrittenExpression.nullable()); + } +} diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 97b9ef86b9910f..e286f09cbda59b 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -507,6 +507,7 @@ struct TQueryOptions { 225: optional i64 runtime_filter_tree_publish_max_send_bytes = 268435456 226: optional bool enable_prune_nested_column = false; + 227: optional bool new_version_bitmap_op_count = false; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 766b791c4ea6b6..a82bc6a0334dee 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -207,6 +207,9 @@ true -- !sql_bitmap_and_count6 -- 0 +-- !sql_bitmap_and_count_nulls -- +0 0 0 + -- !sql_bitmap_or_count1 -- 3 @@ -216,12 +219,21 @@ true -- !sql_bitmap_or_count3 -- 5 +-- !sql_bitmap_or_count_set_diff_length -- +5 + -- !sql_bitmap_or_count4 -- 6 -- !sql_bitmap_or_count5 -- 6 +-- !sql_bitmap_or_count_nulls -- +3 3 0 + +-- !sql_bitmap_or_count_nulls_multi -- +5 + -- !sql -- 2 @@ -255,6 +267,12 @@ true -- !sql_bitmap_xor_count6 -- 0 +-- !sql_bitmap_xor_count_nulls -- +0 0 0 + +-- !sql_bitmap_count_rewrite_nullable -- +3 3 0 0 0 0 + -- !sql_bitmap_and_count7 -- 1 1 2 1 diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index e09f6e02506cbc..900e8571c52e6a 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -294,13 +294,17 @@ suite("test_bitmap_function") { qt_sql_bitmap_and_count4 """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5')) """ qt_sql_bitmap_and_count5 """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),bitmap_empty()) """ qt_sql_bitmap_and_count6 """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """ + qt_sql_bitmap_and_count_nulls """ select bitmap_and_count(cast(NULL as bitmap), bitmap_from_string('1,2,3')), bitmap_and_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)), bitmap_and_count(cast(NULL as bitmap), cast(NULL as bitmap)) """ // bitmap_or_count qt_sql_bitmap_or_count1 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ qt_sql_bitmap_or_count2 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" qt_sql_bitmap_or_count3 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql_bitmap_or_count_set_diff_length """ select bitmap_or_count(bitmap_from_string('1,2'), bitmap_from_string('2,3,4,5')) """ qt_sql_bitmap_or_count4 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ qt_sql_bitmap_or_count5 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ + qt_sql_bitmap_or_count_nulls """ select bitmap_or_count(cast(NULL as bitmap), bitmap_from_string('1,2,3')), bitmap_or_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)), bitmap_or_count(cast(NULL as bitmap), cast(NULL as bitmap)) """ + qt_sql_bitmap_or_count_nulls_multi """ select bitmap_or_count(cast(NULL as bitmap), bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), cast(NULL as bitmap)) """ // BITMAP_XOR qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """ @@ -316,6 +320,8 @@ suite("test_bitmap_function") { qt_sql_bitmap_xor_count4 """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))) """ qt_sql_bitmap_xor_count5 """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())) """ qt_sql_bitmap_xor_count6 """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)) """ + qt_sql_bitmap_xor_count_nulls """ select bitmap_xor_count(cast(NULL as bitmap), bitmap_from_string('1,2,3')), bitmap_xor_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)), bitmap_xor_count(cast(NULL as bitmap), cast(NULL as bitmap)) """ + qt_sql_bitmap_count_rewrite_nullable """ select bitmap_count(bitmap_or(cast(NULL as bitmap), bitmap_from_string('1,2,3'))), bitmap_or_count(cast(NULL as bitmap), bitmap_from_string('1,2,3')), bitmap_count(bitmap_and(bitmap_from_string('1,2,3'), cast(NULL as bitmap))), bitmap_and_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)), bitmap_count(bitmap_xor(bitmap_from_string('1,2,3'), cast(NULL as bitmap))), bitmap_xor_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)) """ // bitmap_and_count, bitmap_xor_count, bitmap_and_not_count of all nullable column sql """ DROP TABLE IF EXISTS test_bitmap1 """