diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java index b3512835bd8d7f..973709b0c2e3bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.BiMap; @@ -56,24 +57,15 @@ * post rule to do lazy materialize */ public class LazyMaterializeTopN extends PlanPostProcessor { - /* BE do not support pattern: - union - -->materialize - -->topn - -->scan1 - -->materialize - -->topn - -->scan2 - when we create materializeNode for the first union child, set hasMaterialized=true - to avoid generating materializeNode for other union's children - */ private static final Logger LOG = LogManager.getLogger(LazyMaterializeTopN.class); - private boolean hasMaterialized = false; @Override public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { + // Visit children first (bottom-up) so that TopN nodes under union are processed independently + Plan topNWithNewChildren = DefaultPlanRewriter.visitChildren(this, topN, ctx); + PhysicalTopN topNToProcess = (PhysicalTopN) topNWithNewChildren; try { - Plan result = computeTopN(topN, ctx); + Plan result = computeTopN(topNToProcess, ctx); if (SessionVariable.isFeDebug()) { Validator validator = new Validator(); validator.processRoot(result, ctx); @@ -81,14 +73,11 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { return result; } catch (Exception e) { LOG.warn("lazy materialize topn failed", e); - return topN; + return topNToProcess; } } private Plan computeTopN(PhysicalTopN topN, CascadesContext ctx) { - if (hasMaterialized) { - return topN; - } if (SessionVariable.getTopNLazyMaterializationThreshold() < topN.getLimit()) { return topN; } @@ -183,7 +172,6 @@ private Plan computeTopN(PhysicalTopN topN, CascadesContext ctx) { result = new PhysicalLazyMaterialize(result, result.getOutput(), materializedSlots, relationToLazySlotMap, relationToRowId, materializeMap, null, ((AbstractPlan) result).getStats()); - hasMaterialized = true; } else { /* topn @@ -206,7 +194,6 @@ private Plan computeTopN(PhysicalTopN topN, CascadesContext ctx) { result = new PhysicalLazyMaterialize(result, materializeInput, reOrderedMaterializedSlots, relationToLazySlotMap, relationToRowId, materializeMap, null, ((AbstractPlan) result).getStats()); - hasMaterialized = true; } result = new PhysicalProject(originOutput, null, result); return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java index d70682961f5046..e0b5fdfd10e40c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey; import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue; @@ -127,9 +128,42 @@ public Void visitSlotReference(SlotReference slotReference, CollectorContext con int slotId = slotReference.getExprId().asInt(); slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type)); } + if (dataType.isStringLikeType()) { + int slotId = slotReference.getExprId().asInt(); + if (!context.accessPathBuilder.isEmpty()) { + // Accessed via an offset-only function (e.g. length()). + // Builder already has "offset" at the tail; add the column name as prefix. + context.accessPathBuilder.addPrefix(slotReference.getName()); + ImmutableList path = ImmutableList.copyOf(context.accessPathBuilder.accessPath); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + } else { + // Direct access to the string column → record a DATA path so that any + // concurrent offset-only path for the same slot is suppressed. + List path = ImmutableList.of(slotReference.getName()); + slotToAccessPaths.put(slotId, + new CollectAccessPathResult(path, context.bottomFilter, TAccessPathType.DATA)); + } + } return null; } + @Override + public Void visitLength(Length length, CollectorContext context) { + Expression arg = length.child(0); + // length() only needs the offset array, not the chars data. + // Add ACCESS_STRING_OFFSET as a suffix so the path builder accumulates + // e.g. ["str_col", "offset"] or ["c_struct", "f3", "offset"]. + if (arg.getDataType().isStringLikeType() && context.accessPathBuilder.isEmpty()) { + CollectorContext offsetContext = + new CollectorContext(context.statementContext, context.bottomFilter); + offsetContext.accessPathBuilder.addSuffix(AccessPathInfo.ACCESS_STRING_OFFSET); + return arg.accept(this, offsetContext); + } + // fall through to default (recurse into children with fresh contexts) + return visit(length, context); + } + @Override public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) { if (nameToLambdaArguments.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java index 2b9a34c21a6114..864d4d69f8e661 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathInfo.java @@ -27,6 +27,9 @@ public class AccessPathInfo { public static final String ACCESS_ALL = "*"; public static final String ACCESS_MAP_KEYS = "KEYS"; public static final String ACCESS_MAP_VALUES = "VALUES"; + // Suffix appended to a string-column path to indicate that only the offset array + // (not the char data) is needed — agreed with BE as the special path component name. + public static final String ACCESS_STRING_OFFSET = "OFFSET"; private DataType prunedType; // allAccessPaths is used to record all access path include predicate access path and non-predicate access path, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java index eb790b447e86bf..a4215730e31dbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java @@ -69,7 +69,9 @@ public Map> collect(Plan root, StatementCont } private boolean shouldCollectAccessPath(Slot slot) { - return slot.getDataType() instanceof NestedColumnPrunable || slot.getDataType().isVariantType(); + return slot.getDataType() instanceof NestedColumnPrunable + || slot.getDataType().isVariantType() + || slot.getDataType().isStringLikeType(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java index f30f7fb93e9fbb..65ef05ca44f704 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Length; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.types.ArrayType; @@ -80,7 +81,9 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { StatementContext statementContext = jobContext.getCascadesContext().getStatementContext(); SessionVariable sessionVariable = statementContext.getConnectContext().getSessionVariable(); if (!sessionVariable.enablePruneNestedColumns - || (!statementContext.hasNestedColumns() && !containsVariant(plan))) { + || (!statementContext.hasNestedColumns() + && !containsVariant(plan) + && !(sessionVariable.enableSubColumnMetaAccess && containsStringLength(plan)))) { return plan; } @@ -104,6 +107,38 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) { } } + /** Returns true when the plan tree contains length() applied to a string-type expression. + * Used in the early-exit guard so that string offset optimizations are not skipped even + * when no nested (struct/array/map) or variant columns are present. */ + private static boolean containsStringLength(Plan plan) { + AtomicBoolean found = new AtomicBoolean(false); + plan.foreachUp(node -> { + if (found.get()) { + return; + } + Plan current = (Plan) node; + for (Expression expression : current.getExpressions()) { + if (expressionContainsStringLength(expression)) { + found.set(true); + return; + } + } + }); + return found.get(); + } + + private static boolean expressionContainsStringLength(Expression expr) { + if (expr instanceof Length && expr.child(0).getDataType().isStringLikeType()) { + return true; + } + for (Expression child : expr.children()) { + if (expressionContainsStringLength(child)) { + return true; + } + } + return false; + } + private static boolean containsVariant(Plan plan) { AtomicBoolean hasVariant = new AtomicBoolean(false); plan.foreachUp(node -> { @@ -183,6 +218,18 @@ private static Map pruneDataType( DataTypeAccessTree accessTree = kv.getValue(); DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType()); + if (slot.getDataType().isStringLikeType()) { + if (accessTree.hasStringOffsetOnlyAccess()) { + // Offset-only access (e.g. length(str_col)): type stays varchar, + // but we must still send the access path to BE so it skips the char data. + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); + result.put(slot.getExprId().asInt(), + new AccessPathInfo(slot.getDataType(), allPaths, new ArrayList<>())); + } + // direct access (accessAll=true) or other: skip — no type change, no access paths needed. + continue; + } + List allPaths = buildColumnAccessPaths(slot, allAccessPaths); result.put(slot.getExprId().asInt(), new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>())); @@ -202,7 +249,9 @@ private static Map pruneDataType( List predicatePaths = buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); - accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + if (accessPathInfo != null) { + accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + } } for (Entry kv : variantSlots.entrySet()) { @@ -210,7 +259,9 @@ private static Map pruneDataType( List predicatePaths = buildColumnAccessPaths(slot, predicateAccessPaths); AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt()); - accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + if (accessPathInfo != null) { + accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths); + } } return result; @@ -271,6 +322,11 @@ public static class DataTypeAccessTree { // if access 's.a.b' the node 's' and 'a' has accessPartialChild, and node 'b' has accessAll private boolean accessPartialChild; private boolean accessAll; + // True when this string-typed node is accessed ONLY via the offset array + // (e.g. length(str_col) or length(element_at(c_struct,'f3'))). + // When this flag is set and accessAll is NOT set, pruneDataType() returns BigIntType + // to signal that the BE only needs to read the offset array, not the chars data. + private boolean isStringOffsetOnly; // for the future, only access the meta of the column, // e.g. `is not null` can only access the column's offset, not need to read the data private TAccessPathType pathType; @@ -312,6 +368,16 @@ public Map getChildren() { return children; } + /** True when the string column is accessed ONLY via the offset array (e.g. length(str_col)), + * meaning the type must not change but an access path still needs to be sent to BE. */ + public boolean hasStringOffsetOnlyAccess() { + if (isRoot) { + DataTypeAccessTree child = children.values().iterator().next(); + return child.type.isStringLikeType() && child.isStringOffsetOnly && !child.accessAll; + } + return type.isStringLikeType() && isStringOffsetOnly && !accessAll; + } + /** pruneCastType */ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) { if (type instanceof StructType) { @@ -447,6 +513,16 @@ public void setAccessByPath(List path, int accessIndex, TAccessPathType valuesChild.setAccessByPath(path, accessIndex + 1, pathType); return; } + } else if (type.isStringLikeType()) { + // String leaf accessed via the offset array (e.g. path ends in "offset"). + // Mark offset-only so pruneDataType() can return BigIntType instead of full data. + if (path.get(accessIndex).equals(AccessPathInfo.ACCESS_STRING_OFFSET)) { + isStringOffsetOnly = true; + return; // do NOT set accessAll — offset-only is distinguishable from full access + } + // Any other sub-path on a string column means full data is needed. + accessAll = true; + return; } else if (isRoot) { children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path, accessIndex + 1, pathType); return; @@ -484,6 +560,10 @@ public Optional pruneDataType() { return children.values().iterator().next().pruneDataType(); } else if (accessAll) { return Optional.of(type); + } else if (isStringOffsetOnly) { + // Only the offset array is accessed (e.g. length(str_col)). + // The slot type stays unchanged (varchar); the access path tells BE to skip char data. + return Optional.empty(); } else if (!accessPartialChild) { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java index e1536918dcb958..068afd626ff6b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java @@ -724,7 +724,8 @@ private void tryRecordReplaceSlots(Plan plan, Object checkObj, Set shou for (Slot slot : output) { int slotId = slot.getExprId().asInt(); if ((slot.getDataType() instanceof NestedColumnPrunable - || slot.getDataType().isVariantType()) + || slot.getDataType().isVariantType() + || slot.getDataType().isStringLikeType()) && replacedDataTypes.containsKey(slotId)) { shouldReplaceSlots.add(slotId); shouldPrune = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index d54a128dcc9ba2..fad3736b4f7178 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -424,6 +424,7 @@ public class SessionVariable implements Serializable, Writable { "enable_runtime_filter_partition_prune"; public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column"; + public static final String ENABLE_SUB_COLUMN_META_ACCESS = "enable_sub_column_meta_access"; static final String SESSION_CONTEXT = "session_context"; @@ -1683,6 +1684,17 @@ public enum IgnoreSplitType { ) public boolean enablePruneNestedColumns = true; + @VariableMgr.VarAttr(name = ENABLE_SUB_COLUMN_META_ACCESS, needForward = true, + fuzzy = false, + varType = VariableAnnotation.EXPERIMENTAL, + description = { + "是否开启子列 meta 访问优化（如 length(str) 只读 offset、IS NULL 只读 null flag）", + "Whether to enable sub-column meta access optimization " + + "(e.g. length(str) reads only offsets, IS NULL reads only null flags)" + } + ) + public boolean enableSubColumnMetaAccess = true; + public boolean enableTopnLazyMaterialization() { return ConnectContext.get() != null && ConnectContext.get().getSessionVariable().topNLazyMaterializationThreshold > 0; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java index 1dfbca127dee47..ae4fe67a802983 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java @@ -100,6 +100,13 @@ public void createTable() throws Exception { + " v variant\n" + ") properties ('replication_num'='1')"); + // Table for string-length offset-only optimization tests + createTable("create table str_tbl(\n" + + " id int,\n" + + " str_col string,\n" + + " c_struct struct\n" + + ") properties ('replication_num'='1')"); + connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name()); connectContext.getSessionVariable().enableNereidsTimeout = false; } @@ -1176,6 +1183,95 @@ private void assertColumns(String sql, } } + // @Test + // public void testStringLengthPruning() { + // // ── Case 1: length(str_col) only ─ offset-only optimization applied ────────── + // assertStringColumn( + // "select length(str_col) from str_tbl", + // "str_col", + // true, + // ImmutableList.of(path("str_col", "offset"))); + + // // ── Case 2: length(str_col) + direct projection of str_col ─ suppressed ───── + // assertStringColumn( + // "select length(str_col), str_col from str_tbl", + // "str_col", + // false, + // ImmutableList.of()); + + // // ── Case 3: length(str_col) + substr(str_col, …) ─ suppressed ─────────────── + // assertStringColumn( + // "select length(str_col), substr(str_col, 2) from str_tbl", + // "str_col", + // false, + // ImmutableList.of()); + + // // ── Case 4: length applied to a struct field ─ struct pruned to bigint field ─ + // // c_struct has {f1:int, f3:string}; only f3 accessed offset-only → + // // pruned type is struct, access path is DATA(["c_struct","f3","offset"]) + // assertColumn( + // "select length(struct_element(c_struct, 'f3')) from str_tbl", + // "struct", + // ImmutableList.of(path("c_struct", "f3", "offset")), + // ImmutableList.of()); + + // // ── Case 5: length(struct field) + direct read of same field ─ suppressed ─── + // // Both the full-data path ["c_struct","f3"] and offset path ["c_struct","f3","offset"] + // // are recorded; f3 pruneDataType() sees accessAll=true → returns text (not bigint). + // assertColumn( + // "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from str_tbl", + // "struct", + // ImmutableList.of(path("c_struct", "f3"), path("c_struct", "f3", "offset")), + // ImmutableList.of()); + // } + + // /** + // * Verify that a specific string-typed column in the rewritten LogicalOlapScan either has + // * BigIntType (offset-only optimization applied) or retains its original string type (suppressed). + // * + // * @param sql query to analyze and rewrite + // * @param columnName name of the string column to inspect + // * @param expectOptimized true → expect BigIntType + access paths; false → expect string type + // * @param expectAllPaths expected access paths when {@code expectOptimized} is true + // */ + // private void assertStringColumn(String sql, String columnName, + // boolean expectOptimized, List expectAllPaths) { + // Plan rewritePlan = PlanChecker.from(connectContext) + // .analyze(sql) + // .rewrite() + // .getCascadesContext() + // .getRewritePlan(); + + // LogicalOlapScan scan = rewritePlan.collect(LogicalOlapScan.class::isInstance) + // .stream() + // .map(p -> (LogicalOlapScan) p) + // .findFirst() + // .orElseThrow(() -> new AssertionError("No LogicalOlapScan in plan for: " + sql)); + + // for (Slot slot : scan.getOutput()) { + // if (!slot.getName().equalsIgnoreCase(columnName)) { + // continue; + // } + // SlotReference slotRef = (SlotReference) slot; + // if (expectOptimized) { + // Assertions.assertEquals(BigIntType.INSTANCE, slotRef.getDataType(), + // "Slot '" + columnName + "' should be BigIntType after offset-only optimization"); + // Optional> allPaths = slotRef.getAllAccessPaths(); + // Assertions.assertTrue(allPaths.isPresent() && !allPaths.get().isEmpty(), + // "Slot '" + columnName + "' should have access paths set"); + // Assertions.assertEquals( + // new TreeSet<>(expectAllPaths), + // new TreeSet<>(allPaths.get()), + // "Unexpected access paths for slot '" + columnName + "'"); + // } else { + // Assertions.assertNotEquals(BigIntType.INSTANCE, slotRef.getDataType(), + // "Slot '" + columnName + "' should NOT be BigIntType (optimization suppressed)"); + // } + // return; + // } + // Assertions.fail("Column '" + columnName + "' not found in LogicalOlapScan output for: " + sql); + // } + private Pair> collectComplexSlots(String sql) throws Exception { NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner(); List complexSlots = new ArrayList<>(); diff --git a/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy new file mode 100644 index 00000000000000..9ff551d921c276 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/column_pruning/string_length_column_pruning.groovy @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Regression tests for the string-length offset-only optimization. +// +// When length() is the *only* use of a string column (or a string field inside a +// struct), the FE should emit a DATA access path with an extra "offset" component so +// that the BE can satisfy the query by reading only the offset array instead of the +// full chars data. The EXPLAIN plan should show: +// nested columns: :[DATA(.offset)] +// +// Crucially, the slot type must remain varchar (not bigint), and any predicate +// using length() must be preserved as-is (e.g. "length(str_col) > 1"), never +// rewritten to "CAST(str_col AS int) > 1". +// +// When the same string column is also read directly (e.g. projected, passed to +// substr(), …) the optimization must be suppressed: no nested-columns entry for +// the plain string column should appear. + +suite("string_length_column_pruning") { + sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" + + sql """ DROP TABLE IF EXISTS slcp_str_tbl """ + sql """ + CREATE TABLE slcp_str_tbl ( + id INT, + str_col STRING, + c_struct STRUCT + ) ENGINE = OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_allocation" = "tag.location.default: 1") + """ + + // ─── Optimizable cases ────────────────────────────────────────────────────── + + // Plain string column: length() is the only use → offset access path emitted, + // slot type stays varchar (not bigint). + explain { + sql "select length(str_col) from slcp_str_tbl" + contains "nested columns" + contains "offset" + notContains "type=bigint" + } + + // Struct string field: length(struct_element) is the only use + explain { + sql "select length(struct_element(c_struct, 'f3')) from slcp_str_tbl" + contains "nested columns" + contains "offset" + notContains "type=bigint" + } + + // length() in both SELECT and WHERE: predicate must remain length(str_col) > 1, + // never be rewritten to CAST(str_col AS int) > 1. Slot type must stay varchar. + explain { + sql "select length(str_col) from slcp_str_tbl where length(str_col) > 1" + contains "nested columns" + contains "offset" + contains "length(str_col" + notContains "CAST(str_col" + notContains "type=bigint" + } + + // ─── Non-optimizable cases ────────────────────────────────────────────────── + + // str_col also projected directly → full chars data needed, offset path suppressed. + // No nested-columns entry for str_col, slot type stays varchar. + explain { + sql "select length(str_col), str_col from slcp_str_tbl" + notContains "type=bigint" + notContains "CAST(str_col" + } + + // str_col also used in substr() → full chars data needed + explain { + sql "select length(str_col), substr(str_col, 2) from slcp_str_tbl" + notContains "type=bigint" + notContains "CAST(str_col" + } + + // Struct field also projected directly → field access is full, not offset-only + // The struct's nested-columns entry still appears (partial struct pruning), + // but the pruned field type must remain text (not bigint). + explain { + sql "select length(struct_element(c_struct, 'f3')), struct_element(c_struct, 'f3') from slcp_str_tbl" + contains "nested columns" + notContains "bigint" + } + +}