HIVE-29538: AssertionError in StatefulFunctionsChecker when compiling queries with NVL/COALESCE/IF over field access on ARRAY<STRUCT> (#6402)

Author: ggangadharan

ql/src/java/org/apache/hadoop/hive/ql/parse/type/HiveFunctionHelper.java

@@ -345,12 +345,7 @@ public Void visitCall(final RexCall call) {
         // doing this check.
         GenericUDF nodeUDF = SqlFunctionConverter.getHiveUDF(
             call.getOperator(), call.getType(), call.getOperands().size());
-        if (nodeUDF == null) {
-          throw new AssertionError("Cannot retrieve function " + call.getOperator().getName()
-              + " within StatefulFunctionsChecker");
-        }
-        // Stateful?
-        if (FunctionRegistry.isStateful(nodeUDF)) {
+        if (nodeUDF != null && FunctionRegistry.isStateful(nodeUDF)) {
           throw new Util.FoundOne(call);
         }
         return super.visitCall(call);

ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestHiveFunctionHelper.java

@@ -18,13 +18,18 @@
 package org.apache.hadoop.hive.ql.parse.type;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveComponentAccess;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 import com.google.common.collect.Lists;
@@ -41,9 +46,9 @@ public void testGetUDTFFunction() throws SemanticException {
     RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
     RexBuilder rexBuilder = new RexBuilder(typeFactory);
     List<RexNode> operands =
-        Lists.newArrayList(rexBuilder.makeLiteral("hello"), rexBuilder.makeLiteral("world"));
+            Lists.newArrayList(rexBuilder.makeLiteral("hello"), rexBuilder.makeLiteral("world"));
     List<RexNode> arrayNode =
-        Lists.newArrayList(rexBuilder.makeCall(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, operands));
+            Lists.newArrayList(rexBuilder.makeCall(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, operands));
 
     FunctionHelper functionHelper = new HiveFunctionHelper(rexBuilder);
     RexCall explodeNode = (RexCall) functionHelper.getUDTFFunction("explode", arrayNode);
@@ -62,4 +67,28 @@ public void testGetUDTFFunctionThrowingException() throws SemanticException {
     // 'upper' is not a udtf so should throw exception
     functionHelper.getUDTFFunction("upper", operands);
   }
+
+  @Test
+  public void testCoalesceWithComponentAccessDoesNotAssert() throws SemanticException {
+    RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+    RexBuilder rexBuilder = new RexBuilder(typeFactory);
+    FunctionHelper functionHelper = new HiveFunctionHelper(rexBuilder);
+
+    // Simulate nested field access over a collection, which is represented in Calcite by
+    // a COMPONENT_ACCESS operator and can appear inside NVL/COALESCE rewrites to CASE.
+    RexNode array =
+            rexBuilder.makeCall(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR,
+                    Lists.newArrayList(rexBuilder.makeLiteral("hello")));
+    RexNode componentAccess =
+            rexBuilder.makeCall(array.getType().getComponentType(), HiveComponentAccess.COMPONENT_ACCESS,
+                    Lists.newArrayList(array));
+
+    FunctionInfo fi = functionHelper.getFunctionInfo("coalesce");
+    List<RexNode> inputs = Lists.newArrayList(componentAccess, rexBuilder.makeNullLiteral(componentAccess.getType()));
+    RexNode expr = functionHelper.getExpression("coalesce", fi, inputs, componentAccess.getType());
+    assertNotNull(expr);
+    assertTrue(expr instanceof RexCall);
+    // COALESCE is rewritten to CASE; the stateful-functions checker walks this tree.
+    assertEquals(SqlKind.CASE, ((RexCall) expr).getOperator().getKind());
+  }
 }

ql/src/test/queries/clientpositive/cbo_component_access_if.q

@@ -0,0 +1,11 @@
+-- HIVE-29538: AssertionError in StatefulFunctionsChecker when Calcite introduces COMPONENT_ACCESS
+
+-- Single-level nesting: array-of-struct field projection uses COMPONENT_ACCESS.
+CREATE TABLE cbo_component_access_if_tbl (
+  `jobs` array<struct<code:string>>
+) STORED AS ORC;
+
+-- `if(...)` is rewritten to CASE and triggers checkForStatefulFunctions.
+EXPLAIN CBO
+SELECT if(concat_ws(',', `jobs`.code) = '', null, concat_ws(',', `jobs`.code)) AS codes
+FROM cbo_component_access_if_tbl;

ql/src/test/results/clientpositive/llap/cbo_component_access_if.q.out

@@ -0,0 +1,28 @@
+PREHOOK: query: CREATE TABLE cbo_component_access_if_tbl (
+  `jobs` array<struct<code:string>>
+) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cbo_component_access_if_tbl
+POSTHOOK: query: CREATE TABLE cbo_component_access_if_tbl (
+  `jobs` array<struct<code:string>>
+) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cbo_component_access_if_tbl
+PREHOOK: query: EXPLAIN CBO
+SELECT if(concat_ws(',', `jobs`.code) = '', null, concat_ws(',', `jobs`.code)) AS codes
+FROM cbo_component_access_if_tbl
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cbo_component_access_if_tbl
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO
+SELECT if(concat_ws(',', `jobs`.code) = '', null, concat_ws(',', `jobs`.code)) AS codes
+FROM cbo_component_access_if_tbl
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cbo_component_access_if_tbl
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(codes=[CASE(=(concat_ws(_UTF-16LE',':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", COMPONENT_ACCESS($0).code), _UTF-16LE''), null:VARCHAR(2147483647) CHARACTER SET "UTF-16LE", concat_ws(_UTF-16LE',':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", COMPONENT_ACCESS($0).code))])
+  HiveTableScan(table=[[default, cbo_component_access_if_tbl]], table:alias=[cbo_component_access_if_tbl])
+