[WIP] HIVE-29641: Upgrade Calcite to 1.42.0

druid-handler/pom.xml

@@ -353,10 +353,15 @@
                       <pattern>io.netty</pattern>
                       <shadedPattern>org.apache.hive.druid.io.netty</shadedPattern>
                     </relocation>
+                    <!-- Calcite is intentionally NOT included or relocated here. Druid 0.17.1 uses Calcite APIs
+                    that are compatible with Hive's Calcite 1.42+. Including calcite-core in the shade caused
+                    SqlFunctions.class to exceed the JVM 64KB method limit after relocation. If Druid is upgraded
+                    to a version with an incompatible Calcite, this relocation must be restored (excluding SqlFunctions,
+                    or splitting the class via a source-level patch to Calcite).
                     <relocation>
                       <pattern>org.apache.calcite</pattern>
                       <shadedPattern>org.apache.hive.druid.org.apache.calcite</shadedPattern>
-                    </relocation>
+                    </relocation> -->
                     <relocation>
                       <pattern>org.jboss.netty</pattern>
                       <shadedPattern>org.apache.hive.druid.org.jboss.netty</shadedPattern>
@@ -379,7 +384,8 @@
                       <include>org.apache.druid:*</include>
                       <include>org.apache.druid.extensions:*</include>
                       <include>io.netty:*</include>
-                      <include>org.apache.calcite:*</include>
+                      <!-- See comment above about Calcite not being relocated & included -->
+                      <!-- <include>org.apache.calcite:*</include> -->
                       <include>com.fasterxml.jackson.core:*</include>
                       <include>com.fasterxml.jackson.datatype:*</include>
                       <include>com.fasterxml.jackson.dataformat:*</include>

hbase-handler/src/test/results/positive/hbase_queries.q.out

@@ -148,7 +148,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: UDFToDouble(key) (type: double)
@@ -604,7 +604,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out

@@ -1458,37 +1458,40 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: srcpart_double_hour_n0
-                  filterExpr: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                  filterExpr: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                   Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                    predicate: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                     Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string)
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        null sort order: z
-                        sort order: +
-                        Map-reduce partition columns: _col0 (type: string)
-                        Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                      Select Operator
-                        expressions: _col0 (type: string)
-                        outputColumnNames: _col0
+                      Filter Operator
+                        predicate: _col0 is not null (type: boolean)
                         Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                        Group By Operator
-                          keys: _col0 (type: string)
-                          minReductionHashAggr: 0.4
-                          mode: hash
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          null sort order: z
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+                        Select Operator
+                          expressions: _col0 (type: string)
                           outputColumnNames: _col0
                           Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                          Dynamic Partitioning Event Operator
-                            Target column: hr (string)
-                            Target Input: srcpart_iceberg
-                            Partition key expr: hr
+                          Group By Operator
+                            keys: _col0 (type: string)
+                            minReductionHashAggr: 0.4
+                            mode: hash
+                            outputColumnNames: _col0
                             Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                            Target Vertex: Map 1
+                            Dynamic Partitioning Event Operator
+                              Target column: hr (string)
+                              Target Input: srcpart_iceberg
+                              Partition key expr: hr
+                              Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+                              Target Vertex: Map 1
             Execution mode: vectorized
         Reducer 2 
             Reduce Operator Tree:
@@ -1587,21 +1590,24 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: srcpart_double_hour_n0
-                  filterExpr: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                  filterExpr: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                   Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                    predicate: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                     Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string)
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        null sort order: z
-                        sort order: +
-                        Map-reduce partition columns: _col0 (type: string)
+                      Filter Operator
+                        predicate: _col0 is not null (type: boolean)
                         Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          null sort order: z
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: vectorized
         Reducer 2 
             Reduce Operator Tree:
@@ -3484,37 +3490,40 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: srcpart_double_hour_n0
-                  filterExpr: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                  filterExpr: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                   Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((UDFToDouble(hour) = 11.0D) and CAST( UDFToInteger((hr / 2.0D)) AS STRING) is not null) (type: boolean)
+                    predicate: ((UDFToDouble(hour) = 11.0D) and hr is not null) (type: boolean)
                     Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: CAST( UDFToInteger((hr / 2.0D)) AS STRING) (type: string)
                       outputColumnNames: _col0
                       Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: string)
-                        null sort order: z
-                        sort order: +
-                        Map-reduce partition columns: _col0 (type: string)
+                      Filter Operator
+                        predicate: _col0 is not null (type: boolean)
                         Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                      Select Operator
-                        expressions: _col0 (type: string)
-                        outputColumnNames: _col0
-                        Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                        Group By Operator
-                          keys: _col0 (type: string)
-                          minReductionHashAggr: 0.4
-                          mode: hash
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          null sort order: z
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+                        Select Operator
+                          expressions: _col0 (type: string)
                           outputColumnNames: _col0
                           Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                          Dynamic Partitioning Event Operator
-                            Target column: hr (string)
-                            Target Input: srcpart_iceberg
-                            Partition key expr: hr
+                          Group By Operator
+                            keys: _col0 (type: string)
+                            minReductionHashAggr: 0.4
+                            mode: hash
+                            outputColumnNames: _col0
                             Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
-                            Target Vertex: Map 1
+                            Dynamic Partitioning Event Operator
+                              Target column: hr (string)
+                              Target Input: srcpart_iceberg
+                              Partition key expr: hr
+                              Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+                              Target Vertex: Map 1
             Execution mode: vectorized
         Reducer 2 
             Execution mode: vectorized

iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out

@@ -621,7 +621,7 @@ STAGE PLANS:
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
                 Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
-                  predicate: ((((_col5 <> _col8) or (_col5 <= 100)) and _col5 is not null) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
+                  predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
                   Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)

itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out

@@ -87,7 +87,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out

@@ -51,7 +51,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out

@@ -105,7 +105,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out

@@ -59,7 +59,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

kudu-handler/src/test/results/positive/kudu_complex_queries.q.out

@@ -85,10 +85,10 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: kv_table
-                  filterExpr: UDFToDouble(key) is not null (type: boolean)
+                  filterExpr: key is not null (type: boolean)
                   Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: UDFToDouble(key) is not null (type: boolean)
+                    predicate: key is not null (type: boolean)
                     Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: UDFToDouble(key) (type: double)
@@ -246,7 +246,7 @@ STAGE PLANS:
                   filterExpr: (key > 100) (type: boolean)
                   Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: (((key % 2) = 0) and UDFToDouble(key) is not null) (type: boolean)
+                    predicate: ((key % 2) = 0) (type: boolean)
                     Statistics: Num rows: 154 Data size: 14630 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

pom.xml

@@ -98,7 +98,7 @@
     <maven.eclipse.plugin.version>2.10</maven.eclipse.plugin.version>
     <maven.exec.plugin.version>3.1.0</maven.exec.plugin.version>
     <maven.versions.plugin.version>2.16.0</maven.versions.plugin.version>
-    <maven.shade.plugin.version>3.6.0</maven.shade.plugin.version>
+    <maven.shade.plugin.version>3.6.2</maven.shade.plugin.version>
     <josdk.version>5.3.4</josdk.version>
     <fabric8.version>7.7.0</fabric8.version>
     <maven.surefire.plugin.version>3.5.3</maven.surefire.plugin.version>
@@ -113,7 +113,7 @@
     <apache-directory-server.version>2.0.0-M24</apache-directory-server.version>
     <avro.version>1.12.0</avro.version>
     <bcprov-jdk18on.version>1.82</bcprov-jdk18on.version>
-    <calcite.version>1.33.0</calcite.version>
+    <calcite.version>1.42.0</calcite.version>
     <datanucleus-api-jdo.version>6.0.5</datanucleus-api-jdo.version>
     <datanucleus-core.version>6.0.11</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>

ql/pom.xml

@@ -365,6 +365,14 @@
       <artifactId>hadoop-yarn-client</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.core5</groupId>
+      <artifactId>httpcore5</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.orc</groupId>
       <artifactId>orc-tools</artifactId>

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.calcite.util.ConversionUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -569,6 +570,10 @@ private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException {
     conf = c;
     // turn off calcite rexnode normalization
     System.setProperty("calcite.enable.rexnode.digest.normalize", "false");
+    // update calcite default charset
+    // TODO is this the right place? It's not considered in cases like TestHiveRelJsonReader
+    System.setProperty("calcite.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
+    System.setProperty("calcite.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
     if (doRegisterAllFns) {
       registerAllFunctionsOnce();
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -48,9 +48,12 @@
 import org.apache.calcite.rex.RexDynamicParam;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
+import org.apache.calcite.rex.RexLambdaRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexPatternFieldRef;
 import org.apache.calcite.rex.RexShuttle;
@@ -1139,6 +1142,21 @@ public Boolean visitSubQuery(RexSubQuery subQuery) {
     public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
       return false;
     }
+
+    @Override
+    public Boolean visitLambda(RexLambda lambda) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+      return false;
+    }
   }
 
   public static Set<Integer> getInputRefs(RexNode expr) {