From 6cc039c51c957110a46d78ce7c89eacf9c26fee6 Mon Sep 17 00:00:00 2001 From: Hari Dara Date: Fri, 10 Apr 2026 11:29:19 +0530 Subject: [PATCH 1/2] HBASE-30110 Fix for flapper in TestZooKeeper by AI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran for 20 times in a loop to verify. AI tool: Claude Code AI Model: Sonnet 4.5 (reverified using Opus 4.6 (1M context)) Prompt: You need to diagnose the HBase test TestZooKeeper that is flapping. It had multiple failures and errors in the PR validation build, but none when I ran locally. The source code is at @../../src/apache/hbase/ and the logs for good run is at @good-run and bad run is at @bad-run. Can you compare the logs and try to understand why the run failed and look at the code for a potential fix? Here is the AI summary for the fix: **Problem Identified:** - Line 78 had a duplicate call to `TEST_UTIL.startMiniDFSCluster(2)` - This violated the "can only create one" constraint and caused HDFS namespace conflicts when tests ran in parallel **Why it was flaky:** - ✅ **Local/isolated runs**: Worked because no leftover HDFS state - ❌ **CI/parallel runs**: Failed when other ZooKeeper tests ran first, leaving behind HDFS data with incompatible `namespaceID` **Fix Applied:** Removed the duplicate call on line 78. The test now properly: 1. Starts Mini DFS Cluster (line 71) 2. Starts Mini ZK Cluster (line 72) 3. Configures settings (lines 73-77) 4. Done! (no duplicate cluster start) This matches the pattern used in other similar tests like `TestClusterBootOrder` and aligns with the comment's intent: "Test we can first start the ZK cluster by itself." The test should now be stable across both local and CI environments. You can verify by running the test suite again to ensure `TestZooKeeper` no longer fails with `InconsistentFSStateException`. --- .../src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java index fc5822477937..8b9736aefb69 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestZooKeeper.java @@ -75,7 +75,6 @@ public static void setUpBeforeClass() throws Exception { conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class, LoadBalancer.class); - TEST_UTIL.startMiniDFSCluster(2); } @AfterAll From dd76adc30cbf4dc689289f73d3b0955fb95d99a7 Mon Sep 17 00:00:00 2001 From: Hari Dara Date: Fri, 24 Apr 2026 22:35:51 +0530 Subject: [PATCH 2/2] Fix for deadlock by AI --- .../master/MetricsMasterWrapperImpl.java | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java index ff6f5b8e5df8..9b134e94b1f7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java @@ -17,18 +17,15 @@ */ package org.apache.hadoop.hbase.master; -import java.io.IOException; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.quotas.QuotaObserverChore; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.util.PairOfSameType; @@ -217,26 +214,22 @@ Entry convertSnapshot(SpaceQuotaSnapshot snapshot) { @Override public PairOfSameType getRegionCounts() { - try { - if (!master.isInitialized()) { - return new PairOfSameType<>(0, 0); + if (!master.isInitialized() || master.getAssignmentManager() == null) { + return new PairOfSameType<>(0, 0); + } + int onlineRegionCount = 0; + int offlineRegionCount = 0; + for (RegionState rs : master.getAssignmentManager().getRegionStates().getRegionStates()) { + if (rs.getRegion().getTable().isSystemTable()) { + continue; } - Integer onlineRegionCount = 0; - Integer offlineRegionCount = 0; - - List descriptors = master.listTableDescriptors(null, null, null, false); - - for (TableDescriptor htDesc : descriptors) { - TableName tableName = htDesc.getTableName(); - Map> tableRegions = - master.getAssignmentManager().getRegionStates().getRegionByStateOfTable(tableName); - onlineRegionCount += tableRegions.get(RegionState.State.OPEN).size(); - offlineRegionCount += tableRegions.get(RegionState.State.OFFLINE).size(); + if (rs.isOpened()) { + onlineRegionCount++; + } else if (rs.isOffline()) { + offlineRegionCount++; } - return new PairOfSameType<>(onlineRegionCount, offlineRegionCount); - } catch (IOException e) { - return new PairOfSameType<>(0, 0); } + return new PairOfSameType<>(onlineRegionCount, offlineRegionCount); } @Override