diff --git a/docs/generated/hive_catalog_configuration.html b/docs/generated/hive_catalog_configuration.html index 63eaab9327e4..0ce809098a4d 100644 --- a/docs/generated/hive_catalog_configuration.html +++ b/docs/generated/hive_catalog_configuration.html @@ -26,6 +26,12 @@ + +
alter-table-cascade
+ true + Boolean + Whether to cascade schema changes to Hive metastore partitions when altering table. +
client-pool-cache.eviction-interval-ms
300000 diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java index 7759a3fd9371..4e183c5812bf 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java @@ -30,23 +30,33 @@ public class HiveAlterTableUtils { public static void alterTable( - IMetaStoreClient client, Identifier identifier, Table table, boolean skipUpdateStats) + IMetaStoreClient client, + Identifier identifier, + Table table, + boolean skipUpdateStats, + boolean cascade) throws TException { try { - alterTableWithEnv(client, identifier, table, skipUpdateStats); + alterTableWithEnv(client, identifier, table, skipUpdateStats, cascade); } catch (NoClassDefFoundError | NoSuchMethodError e) { - alterTableWithoutEnv(client, identifier, table); + alterTableWithoutEnv(client, identifier, table, cascade); } } private static void alterTableWithEnv( - IMetaStoreClient client, Identifier identifier, Table table, boolean skipUpdateStats) + IMetaStoreClient client, + Identifier identifier, + Table table, + boolean skipUpdateStats, + boolean cascade) throws TException { boolean skipHiveUpdateStats = Boolean.parseBoolean(table.getParameters().get(StatsSetupConst.DO_NOT_UPDATE_STATS)) || skipUpdateStats; EnvironmentContext environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.CASCADE, "true"); + if (cascade) { + environmentContext.putToProperties(StatsSetupConst.CASCADE, "true"); + } environmentContext.putToProperties( StatsSetupConst.DO_NOT_UPDATE_STATS, Boolean.toString(skipHiveUpdateStats)); client.alter_table_with_environmentContext( @@ -54,7 +64,8 @@ private static void alterTableWithEnv( } private static void alterTableWithoutEnv( - IMetaStoreClient client, Identifier identifier, Table table) throws TException { - client.alter_table(identifier.getDatabaseName(), identifier.getTableName(), table, true); + IMetaStoreClient client, Identifier identifier, Table table, boolean cascade) + throws TException { + client.alter_table(identifier.getDatabaseName(), identifier.getTableName(), table, cascade); } } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index c988d2b41adb..889850bf1131 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -114,6 +114,7 @@ import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.format.csv.CsvOptions.FIELD_DELIMITER; +import static org.apache.paimon.hive.HiveCatalogOptions.ALTER_TABLE_CASCADE; import static org.apache.paimon.hive.HiveCatalogOptions.HADOOP_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_SKIP_UPDATE_STATS; @@ -1298,7 +1299,11 @@ private void alterTableToHms( .execute( client -> HiveAlterTableUtils.alterTable( - client, identifier, table, skipUpdateStats)); + client, + identifier, + table, + skipUpdateStats, + options.get(ALTER_TABLE_CASCADE))); } @Override diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java index 7700a763d093..9fc16b594662 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java @@ -102,5 +102,12 @@ public final class HiveCatalogOptions { + "E.g. specifying \"conf:a.b.c\" will add \"a.b.c\" to the key, and so that configurations with different default catalog wouldn't share the same client pool. Multiple conf elements can be specified.")) .build()); + public static final ConfigOption ALTER_TABLE_CASCADE = + ConfigOptions.key("alter-table-cascade") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to cascade schema changes to Hive metastore partitions when altering table."); + private HiveCatalogOptions() {} } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java new file mode 100644 index 000000000000..837499b7fa1a --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.catalog.Identifier; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link HiveAlterTableUtils}. */ +public class HiveAlterTableUtilsTest { + + @Test + public void testAlterTableWithoutCascade() throws Exception { + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false); + IMetaStoreClient client = handler.client(); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, false); + + assertThat(handler.environmentContext.getProperties()) + .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") + .doesNotContainKey(StatsSetupConst.CASCADE); + } + + @Test + public void testAlterTableWithCascade() throws Exception { + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false); + IMetaStoreClient client = handler.client(); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, true); + + assertThat(handler.environmentContext.getProperties()) + .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") + .containsEntry(StatsSetupConst.CASCADE, "true"); + } + + @Test + public void testAlterTableWithoutEnvUsesConfiguredCascade() throws Exception { + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(true); + IMetaStoreClient client = handler.client(); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, false); + + assertThat(handler.cascade).isFalse(); + } + + private static class CapturingMetaStoreClient implements InvocationHandler { + + private final boolean failWithEnv; + private EnvironmentContext environmentContext; + private Boolean cascade; + + private CapturingMetaStoreClient(boolean failWithEnv) { + this.failWithEnv = failWithEnv; + } + + private IMetaStoreClient client() { + return (IMetaStoreClient) + Proxy.newProxyInstance( + IMetaStoreClient.class.getClassLoader(), + new Class[] {IMetaStoreClient.class}, + this); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) { + if ("alter_table_with_environmentContext".equals(method.getName())) { + if (failWithEnv) { + throw new NoSuchMethodError(); + } + environmentContext = (EnvironmentContext) args[3]; + return null; + } + if ("alter_table".equals(method.getName()) && args.length == 4) { + cascade = (Boolean) args[3]; + return null; + } + return defaultValue(method.getReturnType()); + } + + private Object defaultValue(Class returnType) { + if (!returnType.isPrimitive() || Void.TYPE.equals(returnType)) { + return null; + } + if (Boolean.TYPE.equals(returnType)) { + return false; + } + if (Character.TYPE.equals(returnType)) { + return '\0'; + } + if (Byte.TYPE.equals(returnType)) { + return (byte) 0; + } + if (Short.TYPE.equals(returnType)) { + return (short) 0; + } + if (Integer.TYPE.equals(returnType)) { + return 0; + } + if (Long.TYPE.equals(returnType)) { + return 0L; + } + if (Float.TYPE.equals(returnType)) { + return 0F; + } + return 0D; + } + } +}