diff --git a/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto similarity index 100% rename from hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto rename to hbase-protocol/src/main/protobuf/RSGroupAdmin.proto diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionMoverWithRSGroupEnable.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionMoverWithRSGroupEnable.java new file mode 100644 index 000000000000..e2842bbc7be0 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionMoverWithRSGroupEnable.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.apache.hadoop.hbase.util.RegionMover; +import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests that RegionMover.unloadRegions() respects RSGroup membership in branch-2: regions + * decommissioned from a server in a non-default RSGroup must land only on other servers in the same + * group, not on servers in unrelated groups. + */ +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) +public class TestRegionMoverWithRSGroupEnable { + + private static final Logger LOG = LoggerFactory.getLogger(TestRegionMoverWithRSGroupEnable.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String TEST_RSGROUP = "test"; + private static final TableName TABLE_NAME = TableName.valueOf("testRegionMoverWithRSGroupEnable"); + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + RSGroupBasedLoadBalancer.class.getName()); + TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + RSGroupAdminEndpoint.class.getName()); + TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 5); + TEST_UTIL.startMiniCluster(5); + } + + @AfterAll + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + // Addresses of the two servers placed in TEST_RSGROUP each test. + private final List
rsservers = new ArrayList<>(2); + // Addresses of the servers that remain in the default group (excludes meta RS). + private final List defaultGroupServers = new ArrayList<>(); + private RSGroupAdminClient rsGroupAdmin; + + @BeforeEach + public void setUp() throws Exception { + rsGroupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection()); + if (rsGroupAdmin.getRSGroupInfo(TEST_RSGROUP) == null) { + rsGroupAdmin.addRSGroup(TEST_RSGROUP); + } + Collection allServers = TEST_UTIL.getAdmin().getRegionServers(); + + // Exclude the RS that hosts hbase:meta to keep the test stable. + ServerName rsContainMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(rs -> !rs.getRegions(TableName.META_TABLE_NAME).isEmpty()).findFirst().get() + .getServerName(); + LOG.info("{} contains hbase:meta, keeping in default group", rsContainMeta); + + // Move any leftover servers back to default before setting up fresh assignments. + RSGroupInfo existingGroup = rsGroupAdmin.getRSGroupInfo(TEST_RSGROUP); + if (existingGroup != null && !existingGroup.getServers().isEmpty()) { + rsGroupAdmin.moveServers(new HashSet<>(existingGroup.getServers()), + RSGroupInfo.DEFAULT_GROUP); + } + + List modifiable = new ArrayList<>(allServers); + modifiable.remove(rsContainMeta); + int i = 0; + for (ServerName server : modifiable) { + if (i == 2) break; + rsservers.add(Address.fromParts(server.getHostname(), server.getPort())); + i++; + } + rsGroupAdmin.moveServers(new HashSet<>(rsservers), TEST_RSGROUP); + LOG.info("Servers moved to {} group: {}", TEST_RSGROUP, rsservers); + + assertEquals(3, rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()); + assertEquals(2, rsGroupAdmin.getRSGroupInfo(TEST_RSGROUP).getServers().size()); + + // Record the three default-group servers (used for isolation assertions). + for (ServerName sn : allServers) { + Address addr = sn.getAddress(); + if (!rsservers.contains(addr)) { + defaultGroupServers.add(sn); + } + } + + if (TEST_UTIL.getAdmin().tableExists(TABLE_NAME)) { + TEST_UTIL.deleteTable(TABLE_NAME); + } + TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(TABLE_NAME) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); + TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes("a"), Bytes.toBytes("z"), 9); + rsGroupAdmin.moveTables(new HashSet<>(Arrays.asList(TABLE_NAME)), TEST_RSGROUP); + TEST_UTIL.waitTableAvailable(TABLE_NAME); + } + + @AfterEach + public void tearDown() throws Exception { + if (TEST_UTIL.getAdmin().tableExists(TABLE_NAME)) { + TEST_UTIL.deleteTable(TABLE_NAME); + } + if (!rsservers.isEmpty()) { + rsGroupAdmin.moveServers(new HashSet<>(rsservers), RSGroupInfo.DEFAULT_GROUP); + } + if (rsGroupAdmin.getRSGroupInfo(TEST_RSGROUP) != null) { + rsGroupAdmin.removeRSGroup(TEST_RSGROUP); + } + rsservers.clear(); + defaultGroupServers.clear(); + } + + /** + * Unloading a server in a non-default RSGroup must move all regions to the remaining server in + * that group — and must not move any region to a server in the default group. + */ + @Test + public void testUnloadRegionsRespectsRSGroup() throws Exception { + Address decommission = rsservers.get(0); + Address online = rsservers.get(1); + String filename = new Path(TEST_UTIL.getDataTestDir(), "testRSGroupUnload").toString(); + + RegionMoverBuilder builder = + new RegionMoverBuilder(decommission.toString(), TEST_UTIL.getConfiguration()); + try (RegionMover rm = builder.filename(filename).ack(true).build()) { + LOG.info("Unloading {}", decommission.getHostname()); + rm.unload(); + } + + HRegionServer onlineRS = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(rs -> rs.getServerName().getAddress().equals(online)).findFirst().get(); + + // Positive assertion: all 9 regions landed on the one remaining test-group server. + assertEquals(9, onlineRS.getNumberOfOnlineRegions(), + "All 9 regions must be on the single remaining server in the test RSGroup"); + + // Isolation assertion: no default-group server received any of the table's regions. + for (ServerName defaultSN : defaultGroupServers) { + HRegionServer defaultRS = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(rs -> rs.getServerName().equals(defaultSN)).findFirst().orElse(null); + if (defaultRS == null) continue; + List tableRegions = defaultRS.getRegions(TABLE_NAME); + assertTrue(tableRegions.isEmpty(), "Default-group server " + defaultSN + + " must not hold any regions of " + TABLE_NAME + " but had: " + tableRegions); + } + } + + /** + * Unloading a server that is in the default RSGroup must still succeed end-to-end when RSGroups + * are enabled. The server's regions should be spread across all available servers (not filtered). + */ + @Test + public void testUnloadDefaultGroupServerWithRSGroupEnabled() throws Exception { + ServerName defaultSN = defaultGroupServers.get(0); + Address decommission = defaultSN.getAddress(); + String filename = new Path(TEST_UTIL.getDataTestDir(), "testDefaultGroupUnload").toString(); + + // Create a table in the default group; the balancer will distribute its 6 regions naturally + // across the 3 default-group servers, so defaultSN will hold at least some. + TableName defaultTable = TableName.valueOf("testDefaultGroupTable"); + if (TEST_UTIL.getAdmin().tableExists(defaultTable)) { + TEST_UTIL.deleteTable(defaultTable); + } + try { + TableDescriptor td = TableDescriptorBuilder.newBuilder(defaultTable) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); + TEST_UTIL.getAdmin().createTable(td, Bytes.toBytes("a"), Bytes.toBytes("z"), 6); + TEST_UTIL.waitTableAvailable(defaultTable); + + RegionMoverBuilder builder = + new RegionMoverBuilder(decommission.toString(), TEST_UTIL.getConfiguration()); + try (RegionMover rm = builder.filename(filename).ack(true).build()) { + LOG.info("Unloading default-group server {}", decommission.getHostname()); + rm.unload(); + } + + // After unload, the decommissioned server must hold no regions of the default table. + HRegionServer decommRS = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(JVMClusterUtil.RegionServerThread::getRegionServer) + .filter(rs -> rs.getServerName().equals(defaultSN)).findFirst().get(); + assertEquals(0, decommRS.getRegions(defaultTable).size(), + "Decommissioned default-group server must hold no regions after unload"); + + // Isolation assertion: no test-group server must hold any region of the default table. + // RegionMover must have restricted move targets to the default group only. + for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster() + .getRegionServerThreads()) { + HRegionServer rs = rst.getRegionServer(); + Address addr = rs.getServerName().getAddress(); + if (rsservers.contains(addr)) { + List found = rs.getRegions(defaultTable); + assertTrue(found.isEmpty(), "Test-group server " + addr + " must not hold any region of " + + defaultTable + " but had: " + found); + } + } + } finally { + if (TEST_UTIL.getAdmin().tableExists(defaultTable)) { + TEST_UTIL.deleteTable(defaultTable); + } + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 6483cc78f4d8..405c32d36ede 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.util; +import com.google.protobuf.ServiceException; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.Closeable; @@ -31,6 +32,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; @@ -67,6 +69,14 @@ import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; @@ -457,7 +467,13 @@ private boolean unloadRegions(boolean unloadFromRack, List isolateRegion try { // Get Online RegionServers List regionServers = new ArrayList<>(); - regionServers.addAll(admin.getRegionServers()); + RSGroupInfo rsgroup = getRSGroupInfo(hostname, port); + if (rsgroup != null) { + LOG.info("{} belongs to RSGroup {}", hostname, rsgroup.getName()); + regionServers.addAll(filterRSGroupServers(rsgroup, admin.getRegionServers())); + } else { + regionServers.addAll(admin.getRegionServers()); + } // Remove the host Region server from target Region Servers list ServerName server = stripServer(regionServers, hostname, port); if (server == null) { @@ -501,6 +517,8 @@ private boolean unloadRegions(boolean unloadFromRack, List isolateRegion if (regionServers.isEmpty()) { LOG.warn("No Regions were moved - no servers available"); return false; + } else { + LOG.info("Available servers {}", regionServers); } unloadRegions(server, regionServers, movedRegions, isolateRegionIdArray); } catch (Exception e) { @@ -838,6 +856,51 @@ private List readServersFromFile(String filename) throws IOException { return servers; } + /** + * Returns the {@link RSGroupInfo} for the given host:port if the RSGroup coprocessor is enabled + * on the cluster, or {@code null} if RSGroups are not in use. + */ + private RSGroupInfo getRSGroupInfo(String host, int port) throws IOException { + if (!admin.getMasterCoprocessorNames().contains("RSGroupAdminEndpoint")) { + return null; + } + RSGroupAdminService.BlockingInterface stub = + RSGroupAdminService.newBlockingStub(admin.coprocessorService()); + GetRSGroupInfoOfServerRequest request = GetRSGroupInfoOfServerRequest.newBuilder() + .setServer(HBaseProtos.ServerName.newBuilder().setHostName(host).setPort(port).build()) + .build(); + try { + GetRSGroupInfoOfServerResponse resp = stub.getRSGroupInfoOfServer(null, request); + if (!resp.hasRSGroupInfo()) { + LOG.debug("No RSGroup found for {}:{} — server may not be registered or address form " + + "(hostname vs IP) may not match what the RS registered with", host, port); + return null; + } + RSGroupProtos.RSGroupInfo proto = resp.getRSGroupInfo(); + // Only name and server membership are needed for filtering; tables/configuration are omitted. + RSGroupInfo rsGroupInfo = new RSGroupInfo(proto.getName()); + for (HBaseProtos.ServerName sn : proto.getServersList()) { + rsGroupInfo.addServer(Address.fromParts(sn.getHostName(), sn.getPort())); + } + return rsGroupInfo; + } catch (ServiceException e) { + throw ProtobufUtil.handleRemoteException(e); + } + } + + @InterfaceAudience.Private + Collection filterRSGroupServers(RSGroupInfo rsgroup, + Collection onlineServers) { + List result = new ArrayList<>(rsgroup.getServers().size()); + for (ServerName server : onlineServers) { + Address address = Address.fromParts(server.getHostname(), server.getPort()); + if (rsgroup.containsServer(address)) { + result.add(server); + } + } + return result; + } + /** * Designates or excludes the servername whose hostname and port portion matches the list given in * the file. Example:
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java new file mode 100644 index 000000000000..add2cb083793 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Unit tests for {@link RegionMover#filterRSGroupServers}. Spins up a 2-node mini cluster so we can + * build a real RegionMover instance; the method under test is pure in-memory logic. + */ +@Tag(MiscTests.TAG) +@Tag(MediumTests.TAG) +public class TestRegionMoverFilterRSGroupServers { + + private static final Logger LOG = + LoggerFactory.getLogger(TestRegionMoverFilterRSGroupServers.class); + + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(2); + } + + @AfterAll + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private RegionMover buildMover() throws Exception { + ServerName any = TEST_UTIL.getAdmin().getRegionServers().iterator().next(); + return new RegionMoverBuilder(any.getHostname() + ":" + any.getPort(), + TEST_UTIL.getConfiguration()).build(); + } + + /** + * When all online servers are members of the default group (as the coprocessor returns), the + * filter must return all of them — no server is dropped. + */ + @Test + public void testDefaultGroupReturnsAllServers() throws Exception { + try (RegionMover rm = buildMover()) { + List allServers = new ArrayList<>(TEST_UTIL.getAdmin().getRegionServers()); + + RSGroupInfo defaultGroup = new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP); + for (ServerName sn : allServers) { + defaultGroup.addServer(Address.fromParts(sn.getHostname(), sn.getPort())); + } + + Collection result = rm.filterRSGroupServers(defaultGroup, allServers); + assertEquals(allServers.size(), result.size()); + assertTrue(result.containsAll(allServers)); + } + } + + /** A non-default group with one member must return only that member. */ + @Test + public void testNonDefaultGroupFiltersToMembers() throws Exception { + try (RegionMover rm = buildMover()) { + List allServers = new ArrayList<>(TEST_UTIL.getAdmin().getRegionServers()); + + ServerName member = allServers.get(0); + RSGroupInfo group = new RSGroupInfo("testgroup"); + group.addServer(Address.fromParts(member.getHostname(), member.getPort())); + + Collection result = rm.filterRSGroupServers(group, allServers); + assertEquals(1, result.size()); + assertTrue(result.contains(member)); + + RSGroupInfo defaultGroup = new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP); + Set defaultGroupRs = + new HashSet<>(rm.filterRSGroupServers(defaultGroup, allServers)); + Assertions.assertFalse(defaultGroupRs.contains(member), + "Default group should not contain test group member"); + + } + } + + /** A non-default group with no matching members must return an empty list. */ + @Test + public void testNonDefaultGroupWithNoMatchReturnsEmpty() throws Exception { + try (RegionMover rm = buildMover()) { + List allServers = new ArrayList<>(TEST_UTIL.getAdmin().getRegionServers()); + + // A group with a server that is not in the live cluster + RSGroupInfo group = new RSGroupInfo("emptygroup"); + group.addServer(Address.fromParts("nonexistent.host", 9999)); + + Collection result = rm.filterRSGroupServers(group, allServers); + assertTrue(result.isEmpty()); + } + } +}