From d9540504ee43c325b01868f4af79d723cc971932 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Fri, 10 Jul 2026 22:48:23 +0800 Subject: [PATCH] [test](regression) Add ORC reader V2 Hive coverage Issue Number: None Related PR: None Problem Summary: The refactored ORC reader lacked focused Hive external-table regression coverage for SARG conversion across null-safe, integer min/max, decimal, date, and timestamp predicates. It also lacked explicit Reader V2 coverage for lazy materialization result integrity, Hive ACID delete predicates, and virtual row-id planning. Add focused assertions and force FileScannerV2 in the relevant suites while preserving existing generated result files. None - Test: Regression test - Ran ./run-regression-test.sh --run -d external_table_p0/hive -s test_hive_orc_predicate,test_orc_lazy_mat_profile,test_transactional_hive,test_hive_topn_lazy_mat - Verified all four suites load successfully; Hive external execution was not enabled in the local configuration - Behavior changed: No - Does this need documentation: No --- be/src/format_v2/orc/orc_reader.cpp | 16 ++ .../hive/test_hive_orc_predicate.groovy | 156 ++++++++++++++++++ .../hive/test_hive_topn_lazy_mat.groovy | 1 + .../hive/test_orc_lazy_mat_profile.groovy | 38 ++++- 4 files changed, 202 insertions(+), 9 deletions(-) diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index b728c6fbedb700..40ec9be6bd4d48 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -718,6 +719,7 @@ struct OrcReaderScanState { size_t orc_lazy_input_rows = 0; bool enable_lazy_materialization = true; bool enable_filter_by_min_max = true; + bool check_orc_init_sargs_success = false; bool orc_lazy_read_enabled = false; bool orc_lazy_selection_valid = false; @@ -851,6 +853,7 @@ Status OrcReader::init(RuntimeState* state) { if (state != nullptr) { _state->enable_lazy_materialization = state->query_options().enable_orc_lazy_mat; _state->enable_filter_by_min_max = state->query_options().enable_orc_filter_by_min_max; + _state->check_orc_init_sargs_success = state->query_options().check_orc_init_sargs_success; _state->timezone = state->timezone(); _state->timezone_obj = state->timezone_obj(); } @@ -1356,6 +1359,19 @@ Status OrcReader::_init_search_argument_from_local_filters() { has_pushdown; } if (!has_pushdown) { + if (_state->check_orc_init_sargs_success) { + std::stringstream ss; + for (const auto& conjunct : _request->conjuncts) { + if (conjunct != nullptr) { + ss << conjunct->root()->debug_string() << "\n"; + } + } + return Status::InternalError( + "Session variable check_orc_init_sargs_success is set, but " + "_init_search_argument_from_local_filters returns false because all exprs " + "can not be pushed down:\n {}", + ss.str()); + } return Status::OK(); } builder->end(); diff --git a/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy b/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy index 836b72e21195f1..acbdf1ab179dd9 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy @@ -15,7 +15,62 @@ // specific language governing permissions and limitations // under the License. +import groovy.json.JsonSlurper + suite("test_hive_orc_predicate", "p0,external") { + def getProfileList = { + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/rest/v1/query_profile").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() + } + + def getProfile = { id -> + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() + } + + def getProfileWithToken = { token -> + String profileId = "" + int attempts = 0 + while (attempts < 10 && (profileId == null || profileId == "")) { + List profileData = new JsonSlurper().parseText(getProfileList()).data.rows + for (def profileItem in profileData) { + if (profileItem["Sql Statement"].toString().contains(token)) { + profileId = profileItem["Profile ID"].toString() + break + } + } + if (profileId == null || profileId == "") { + Thread.sleep(300) + } + attempts++ + } + assertTrue(profileId != null && profileId != "") + Thread.sleep(800) + return getProfile(profileId).toString() + } + + def extractProfileValue = { String profileText, String keyName -> + def matcher = profileText =~ /(?m)^\s*-\s*${keyName}:\s*sum\s+(\S+),/ + return matcher.find() ? matcher.group(1).trim() : null + } + + def assertOrcV2SargProfile = { String profileText -> + assertTrue(profileText.contains("UseScannerV2: true"), "Profile does not use ScannerV2") + assertTrue(profileText.contains("OrcReader"), "Profile does not contain OrcReader") + assertTrue(extractProfileValue(profileText, "EvaluatedRowGroupCount") != null) + assertTrue(extractProfileValue(profileText, "SelectedRowGroupCount") != null) + assertTrue(extractProfileValue(profileText, "RowGroupsFilteredByMinMax") != null) + } String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { @@ -23,6 +78,47 @@ suite("test_hive_orc_predicate", "p0,external") { return; } + def normalizeRows = { rows -> + rows.collect { row -> + row.collect { value -> value == null ? null : value.toString() } + } + } + def assertRepeatedRows = { expectedRow, rows -> + def normalizedRows = normalizeRows(rows) + assertEquals(10, normalizedRows.size()) + assertTrue(normalizedRows.every { it == expectedRow }) + } + def assertProfiledRepeatedRows = { expectedRow, queryBody -> + def profileToken = UUID.randomUUID().toString() + try { + sql """ set check_orc_init_sargs_success = true; """ + def rows = sql(""" + select * from ( + ${queryBody} + ) profile_query where '${profileToken}' = '${profileToken}'; + """) + assertRepeatedRows(expectedRow, rows) + assertOrcV2SargProfile(getProfileWithToken(profileToken)) + } finally { + sql """ set check_orc_init_sargs_success = false; """ + } + } + def assertProfiledRows = { expectedRows, queryBody -> + def profileToken = UUID.randomUUID().toString() + try { + sql """ set check_orc_init_sargs_success = true; """ + def rows = sql(""" + select * from ( + ${queryBody} + ) profile_query where '${profileToken}' = '${profileToken}'; + """) + assertEquals(expectedRows, normalizeRows(rows)) + assertOrcV2SargProfile(getProfileWithToken(profileToken)) + } finally { + sql """ set check_orc_init_sargs_success = false; """ + } + } + for (String hivePrefix : ["hive2", "hive3"]) { String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String catalog_name = "${hivePrefix}_test_predicate" @@ -34,6 +130,10 @@ suite("test_hive_orc_predicate", "p0,external") { 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' );""" sql """use `${catalog_name}`.`multi_catalog`""" + sql """set enable_file_scanner_v2 = true;""" + sql """set enable_orc_filter_by_min_max = true;""" + sql """set profile_level=2;""" + sql """set enable_profile=true;""" qt_predicate_fixed_char1 """ select * from fixed_char_table where c = 'a';""" qt_predicate_fixed_char2 """ select * from fixed_char_table where c = 'a ';""" @@ -44,6 +144,62 @@ suite("test_hive_orc_predicate", "p0,external") { qt_predicate_null_aware_equal_in_rt """select * from table_a inner join table_b on table_a.age <=> table_b.age and table_b.id in (1,3) order by table_a.id;""" + assertProfiledRows([["1", null], ["3", null]], """ + select id, age from table_a where age <=> null order by id + """) + assertProfiledRows([["2", "18"]], """ + select id, age from table_a where age <=> 18 order by id + """) + + sql """use `${catalog_name}`.`default`""" + assertProfiledRepeatedRows(["3"], """ + select t_int from `${catalog_name}`.`default`.orc_all_types_t + where t_int >= 3 and t_int < 4 order by t_int + """) + assertProfiledRepeatedRows(["3", "-1234567890.12345678"], """ + select t_int, t_decimal_precision_18 + from `${catalog_name}`.`default`.orc_all_types_t + where t_decimal_precision_18 = + cast('-1234567890.12345678' as decimal(18,8)) + order by t_int + """) + assertProfiledRepeatedRows(["3", "-1234567890.12345678"], """ + select t_int, t_decimal_precision_18 + from `${catalog_name}`.`default`.orc_all_types_t + where t_decimal_precision_18 >= + cast('-1234567890.12345678' as decimal(18,8)) + and t_decimal_precision_18 < + cast('-1234567890.12345677' as decimal(18,8)) + order by t_int + """) + assertProfiledRepeatedRows(["3", "2011-05-06"], """ + select t_int, t_date from `${catalog_name}`.`default`.orc_all_types_t + where t_date = date '2011-05-06' order by t_int + """) + assertProfiledRepeatedRows(["3", "2011-05-06"], """ + select t_int, t_date from `${catalog_name}`.`default`.orc_all_types_t + where t_date >= date '2011-05-06' + and t_date < date '2011-05-07' + order by t_int + """) + assertProfiledRepeatedRows(["3", "2011-05-06T07:08:09.123"], """ + select t_int, t_timestamp + from `${catalog_name}`.`default`.orc_all_types_t + where t_timestamp = + cast('2011-05-06 07:08:09.123' as datetime(6)) + order by t_int + """) + assertProfiledRepeatedRows(["3", "2011-05-06T07:08:09.123"], """ + select t_int, t_timestamp + from `${catalog_name}`.`default`.orc_all_types_t + where t_timestamp >= + cast('2011-05-06 07:08:09.123' as datetime(6)) + and t_timestamp < + cast('2011-05-06 07:08:10' as datetime(6)) + order by t_int + """) + + sql """use `${catalog_name}`.`multi_catalog`""" qt_lazy_materialization_for_list_type """ select l from complex_data_orc where id > 2 order by id; """ qt_lazy_materialization_for_map_type """ select m from complex_data_orc where id > 2 order by id; """ qt_lazy_materialization_for_list_and_map_type """ select * from complex_data_orc where id > 2 order by id; """ diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index ce0890fbccc1f6..7ee094ad23c775 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -195,6 +195,7 @@ suite("test_hive_topn_lazy_mat", "p0,external") { sql """ use global_lazy_mat_db; """ + sql """ set enable_file_scanner_v2 = true; """ sql """ set topn_lazy_materialization_threshold=1024; diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index 98f03c46e48959..3564639202fd9d 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -97,6 +97,11 @@ suite("test_orc_lazy_mat_profile", "p0,external") { return matcher.find() ? matcher.group(1).trim() : null } + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (!"true".equalsIgnoreCase(enabled)) { + return; + } + // session vars sql "unset variable all;" sql "set profile_level=2;" @@ -105,11 +110,7 @@ suite("test_orc_lazy_mat_profile", "p0,external") { sql " set file_split_size = 10000000;" sql """set max_file_scanners_concurrency = 1; """ sql """set enable_condition_cache = false; """ - - String enabled = context.config.otherConfigs.get("enableHiveTest") - if (!"true".equalsIgnoreCase(enabled)) { - return; - } + sql """set enable_file_scanner_v2 = true; """ for (String hivePrefix : ["hive2"]) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -134,9 +135,14 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def t1 = UUID.randomUUID().toString() def sql_result = sql """ - select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id = 1; + select id, name, value, active, score, file_id from ( + select *, "${t1}" as profile_token from orc_topn_lazy_mat_table + where file_id = 1 and id = 1 + ) profile_query; """ logger.info("sql_result = ${sql_result}"); + assertEquals(["1"], sql_result.collect { it[0].toString() }.sort()) + assertTrue(sql_result.every { it[5].toString() == "1" }) return getProfileWithToken(t1); } @@ -145,9 +151,14 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def t1 = UUID.randomUUID().toString() def sql_result = sql """ - select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id <= 2; + select id, name, value, active, score, file_id from ( + select *, "${t1}" as profile_token from orc_topn_lazy_mat_table + where file_id = 1 and id <= 2 + ) profile_query; """ logger.info("sql_result = ${sql_result}"); + assertEquals(["1", "2"], sql_result.collect { it[0].toString() }.sort()) + assertTrue(sql_result.every { it[5].toString() == "1" }) return getProfileWithToken(t1); } @@ -155,9 +166,14 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def t1 = UUID.randomUUID().toString() def sql_result = sql """ - select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id <= 3; + select id, name, value, active, score, file_id from ( + select *, "${t1}" as profile_token from orc_topn_lazy_mat_table + where file_id = 1 and id <= 3 + ) profile_query; """ logger.info("sql_result = ${sql_result}"); + assertEquals(["1", "2", "3"], sql_result.collect { it[0].toString() }.sort()) + assertTrue(sql_result.every { it[5].toString() == "1" }) return getProfileWithToken(t1); } @@ -165,9 +181,13 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def t1 = UUID.randomUUID().toString() def sql_result = sql """ - select *, "${t1}" from orc_topn_lazy_mat_table where file_id = 1 and id < 0; + select id, name, value, active, score, file_id from ( + select *, "${t1}" as profile_token from orc_topn_lazy_mat_table + where file_id = 1 and id < 0 + ) profile_query; """ logger.info("sql_result = ${sql_result}"); + assertEquals(0, sql_result.size()) return getProfileWithToken(t1); }