From 4abddc6b2cf04be980007ebefe5680f6b9a31a93 Mon Sep 17 00:00:00 2001 From: wzx140 Date: Thu, 11 Jun 2026 12:56:29 +0800 Subject: [PATCH 1/2] [hive] Add catalog option to control alter table cascade --- .../generated/hive_catalog_configuration.html | 6 ++ .../paimon/hive/HiveAlterTableUtils.java | 18 ++++- .../org/apache/paimon/hive/HiveCatalog.java | 7 +- .../paimon/hive/HiveCatalogOptions.java | 7 ++ .../paimon/hive/HiveAlterTableUtilsTest.java | 80 +++++++++++++++++++ 5 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java diff --git a/docs/generated/hive_catalog_configuration.html b/docs/generated/hive_catalog_configuration.html index 63eaab9327e4..0ce809098a4d 100644 --- a/docs/generated/hive_catalog_configuration.html +++ b/docs/generated/hive_catalog_configuration.html @@ -26,6 +26,12 @@ + +
alter-table-cascade
+ true + Boolean + Whether to cascade schema changes to Hive metastore partitions when altering table. +
client-pool-cache.eviction-interval-ms
300000 diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java index 7759a3fd9371..4e98ee00c848 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java @@ -30,23 +30,33 @@ public class HiveAlterTableUtils { public static void alterTable( - IMetaStoreClient client, Identifier identifier, Table table, boolean skipUpdateStats) + IMetaStoreClient client, + Identifier identifier, + Table table, + boolean skipUpdateStats, + boolean cascade) throws TException { try { - alterTableWithEnv(client, identifier, table, skipUpdateStats); + alterTableWithEnv(client, identifier, table, skipUpdateStats, cascade); } catch (NoClassDefFoundError | NoSuchMethodError e) { alterTableWithoutEnv(client, identifier, table); } } private static void alterTableWithEnv( - IMetaStoreClient client, Identifier identifier, Table table, boolean skipUpdateStats) + IMetaStoreClient client, + Identifier identifier, + Table table, + boolean skipUpdateStats, + boolean cascade) throws TException { boolean skipHiveUpdateStats = Boolean.parseBoolean(table.getParameters().get(StatsSetupConst.DO_NOT_UPDATE_STATS)) || skipUpdateStats; EnvironmentContext environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.CASCADE, "true"); + if (cascade) { + environmentContext.putToProperties(StatsSetupConst.CASCADE, "true"); + } environmentContext.putToProperties( StatsSetupConst.DO_NOT_UPDATE_STATS, Boolean.toString(skipHiveUpdateStats)); client.alter_table_with_environmentContext( diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index c988d2b41adb..889850bf1131 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -114,6 +114,7 @@ import static org.apache.paimon.catalog.CatalogUtils.listPartitionsFromFileSystem; import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.format.csv.CsvOptions.FIELD_DELIMITER; +import static org.apache.paimon.hive.HiveCatalogOptions.ALTER_TABLE_CASCADE; import static org.apache.paimon.hive.HiveCatalogOptions.HADOOP_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_SKIP_UPDATE_STATS; @@ -1298,7 +1299,11 @@ private void alterTableToHms( .execute( client -> HiveAlterTableUtils.alterTable( - client, identifier, table, skipUpdateStats)); + client, + identifier, + table, + skipUpdateStats, + options.get(ALTER_TABLE_CASCADE))); } @Override diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java index 7700a763d093..9fc16b594662 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java @@ -102,5 +102,12 @@ public final class HiveCatalogOptions { + "E.g. specifying \"conf:a.b.c\" will add \"a.b.c\" to the key, and so that configurations with different default catalog wouldn't share the same client pool. Multiple conf elements can be specified.")) .build()); + public static final ConfigOption ALTER_TABLE_CASCADE = + ConfigOptions.key("alter-table-cascade") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to cascade schema changes to Hive metastore partitions when altering table."); + private HiveCatalogOptions() {} } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java new file mode 100644 index 000000000000..902fb7ab4887 --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.catalog.Identifier; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** Tests for {@link HiveAlterTableUtils}. */ +public class HiveAlterTableUtilsTest { + + @Test + public void testAlterTableWithoutCascade() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, false); + + EnvironmentContext context = captureEnvironmentContext(client, table); + assertThat(context.getProperties()) + .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") + .doesNotContainKey(StatsSetupConst.CASCADE); + } + + @Test + public void testAlterTableWithCascade() throws Exception { + IMetaStoreClient client = mock(IMetaStoreClient.class); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, true); + + EnvironmentContext context = captureEnvironmentContext(client, table); + assertThat(context.getProperties()) + .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") + .containsEntry(StatsSetupConst.CASCADE, "true"); + } + + private EnvironmentContext captureEnvironmentContext(IMetaStoreClient client, Table table) + throws Exception { + ArgumentCaptor captor = + ArgumentCaptor.forClass(EnvironmentContext.class); + verify(client) + .alter_table_with_environmentContext( + eq("db"), eq("tbl"), same(table), captor.capture()); + return captor.getValue(); + } +} From a49ed48077dfab422bc0b402a03ff0ccee1113cc Mon Sep 17 00:00:00 2001 From: wzx140 Date: Thu, 11 Jun 2026 19:41:20 +0800 Subject: [PATCH 2/2] [hive] Honor cascade option in alter table fallback --- .../paimon/hive/HiveAlterTableUtils.java | 7 +- .../paimon/hive/HiveAlterTableUtilsTest.java | 102 ++++++++++++++---- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java index 4e98ee00c848..4e183c5812bf 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java @@ -39,7 +39,7 @@ public static void alterTable( try { alterTableWithEnv(client, identifier, table, skipUpdateStats, cascade); } catch (NoClassDefFoundError | NoSuchMethodError e) { - alterTableWithoutEnv(client, identifier, table); + alterTableWithoutEnv(client, identifier, table, cascade); } } @@ -64,7 +64,8 @@ private static void alterTableWithEnv( } private static void alterTableWithoutEnv( - IMetaStoreClient client, Identifier identifier, Table table) throws TException { - client.alter_table(identifier.getDatabaseName(), identifier.getTableName(), table, true); + IMetaStoreClient client, Identifier identifier, Table table, boolean cascade) + throws TException { + client.alter_table(identifier.getDatabaseName(), identifier.getTableName(), table, cascade); } } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java index 902fb7ab4887..837499b7fa1a 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveAlterTableUtilsTest.java @@ -25,56 +25,120 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.Table; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.util.HashMap; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; /** Tests for {@link HiveAlterTableUtils}. */ public class HiveAlterTableUtilsTest { @Test public void testAlterTableWithoutCascade() throws Exception { - IMetaStoreClient client = mock(IMetaStoreClient.class); + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false); + IMetaStoreClient client = handler.client(); Identifier identifier = Identifier.create("db", "tbl"); Table table = new Table(); table.setParameters(new HashMap<>()); HiveAlterTableUtils.alterTable(client, identifier, table, true, false); - EnvironmentContext context = captureEnvironmentContext(client, table); - assertThat(context.getProperties()) + assertThat(handler.environmentContext.getProperties()) .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") .doesNotContainKey(StatsSetupConst.CASCADE); } @Test public void testAlterTableWithCascade() throws Exception { - IMetaStoreClient client = mock(IMetaStoreClient.class); + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(false); + IMetaStoreClient client = handler.client(); Identifier identifier = Identifier.create("db", "tbl"); Table table = new Table(); table.setParameters(new HashMap<>()); HiveAlterTableUtils.alterTable(client, identifier, table, true, true); - EnvironmentContext context = captureEnvironmentContext(client, table); - assertThat(context.getProperties()) + assertThat(handler.environmentContext.getProperties()) .containsEntry(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") .containsEntry(StatsSetupConst.CASCADE, "true"); } - private EnvironmentContext captureEnvironmentContext(IMetaStoreClient client, Table table) - throws Exception { - ArgumentCaptor captor = - ArgumentCaptor.forClass(EnvironmentContext.class); - verify(client) - .alter_table_with_environmentContext( - eq("db"), eq("tbl"), same(table), captor.capture()); - return captor.getValue(); + @Test + public void testAlterTableWithoutEnvUsesConfiguredCascade() throws Exception { + CapturingMetaStoreClient handler = new CapturingMetaStoreClient(true); + IMetaStoreClient client = handler.client(); + Identifier identifier = Identifier.create("db", "tbl"); + Table table = new Table(); + table.setParameters(new HashMap<>()); + + HiveAlterTableUtils.alterTable(client, identifier, table, true, false); + + assertThat(handler.cascade).isFalse(); + } + + private static class CapturingMetaStoreClient implements InvocationHandler { + + private final boolean failWithEnv; + private EnvironmentContext environmentContext; + private Boolean cascade; + + private CapturingMetaStoreClient(boolean failWithEnv) { + this.failWithEnv = failWithEnv; + } + + private IMetaStoreClient client() { + return (IMetaStoreClient) + Proxy.newProxyInstance( + IMetaStoreClient.class.getClassLoader(), + new Class[] {IMetaStoreClient.class}, + this); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) { + if ("alter_table_with_environmentContext".equals(method.getName())) { + if (failWithEnv) { + throw new NoSuchMethodError(); + } + environmentContext = (EnvironmentContext) args[3]; + return null; + } + if ("alter_table".equals(method.getName()) && args.length == 4) { + cascade = (Boolean) args[3]; + return null; + } + return defaultValue(method.getReturnType()); + } + + private Object defaultValue(Class returnType) { + if (!returnType.isPrimitive() || Void.TYPE.equals(returnType)) { + return null; + } + if (Boolean.TYPE.equals(returnType)) { + return false; + } + if (Character.TYPE.equals(returnType)) { + return '\0'; + } + if (Byte.TYPE.equals(returnType)) { + return (byte) 0; + } + if (Short.TYPE.equals(returnType)) { + return (short) 0; + } + if (Integer.TYPE.equals(returnType)) { + return 0; + } + if (Long.TYPE.equals(returnType)) { + return 0L; + } + if (Float.TYPE.equals(returnType)) { + return 0F; + } + return 0D; + } } }