HDDS-14076. Disable SSTFilteringService when defrag feature is enabled (#9495)

Author: smengcl

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -376,6 +376,15 @@ public void start(OzoneConfiguration configuration) {
    */
   public void startSnapshotSstFilteringService(OzoneConfiguration conf) {
     if (isSstFilteringSvcEnabled()) {
+      if (isDefragSvcEnabled()) {
+        LOG.info("SstFilteringService is disabled (despite configuration intending to enable it) " +
+            "because SnapshotDefragService is enabled. Defrag effectively performs filtering already.");
+        return;
+      }
+
+      LOG.info("SstFilteringService is enabled. Note SstFilteringService is " +
+          "deprecated in favor of SnapshotDefragService and may be removed in a future release.");
+
       long serviceInterval = conf.getTimeDuration(
           OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL,
           OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT,

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java

@@ -48,6 +48,7 @@
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
 import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager;
 import org.apache.ratis.util.UncheckedAutoCloseable;
 import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
 import org.slf4j.Logger;
@@ -132,6 +133,32 @@ private boolean isSnapshotDeleted(SnapshotInfo snapshotInfo) {
       return snapshotInfo == null || snapshotInfo.getSnapshotStatus() == SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED;
     }
 
+    /**
+     * Checks if the snapshot has been defragged.
+     * @param snapshotInfo snapshotInfo
+     * @return true if the snapshot has been defragged, false otherwise
+     */
+    private boolean isSnapshotDefragged(SnapshotInfo snapshotInfo) {
+      try {
+        OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager();
+        if (omSnapshotManager == null) {
+          return false;
+        }
+        OmSnapshotLocalDataManager localDataManager = omSnapshotManager.getSnapshotLocalDataManager();
+        if (localDataManager == null) {
+          return false;
+        }
+        try (OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider provider =
+                 localDataManager.getOmSnapshotLocalData(snapshotInfo)) {
+          // If snapshot local data version is not 0, it means the snapshot has been defragged
+          return provider.getVersion() > 0;
+        }
+      } catch (IOException e) {
+        LOG.debug("Error checking if snapshot {} is defragged", snapshotInfo.getSnapshotId(), e);
+        return false;
+      }
+    }
+
     /**
      * Marks the snapshot as SSTFiltered by creating a file in snapshot directory.
      * @param snapshotInfo snapshotInfo
@@ -187,6 +214,12 @@ public BackgroundTaskResult call() throws Exception {
               continue;
             }
 
+            // Skip defragged snapshots as defrag already performs filtering
+            if (isSnapshotDefragged(snapshotInfo)) {
+              LOG.debug("Skipping SST filtering for defragged snapshot: {}", snapShotTableKey);
+              continue;
+            }
+
             LOG.debug("Processing snapshot {} to filter relevant SST Files",
                 snapShotTableKey);
             TablePrefixInfo bucketPrefixInfo =

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java

@@ -788,6 +788,14 @@ public boolean needsDefrag() {
       return false;
     }
 
+    /**
+     * Returns the version of the snapshot local data.
+     * @return Version of the snapshot local data
+     */
+    public long getVersion() {
+      return snapshotLocalData.getVersion();
+    }
+
     @Override
     public void close() throws IOException {
       if (previousLock != null) {

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSstFilteringService.java

@@ -21,6 +21,12 @@
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
 import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_PORT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
 import static org.apache.ozone.test.LambdaTestUtils.await;
@@ -508,4 +514,48 @@ private void deleteSnapshot(String volumeName, String bucketName, String snapsho
     writeClient.deleteSnapshot(volumeName, bucketName, snapshotName);
     countTotalSnapshots--;
   }
+
+  /**
+   * Test to verify that SSTFilteringService is disabled when defrag service is enabled.
+   * This test creates a new OzoneManager instance with defrag service enabled and verifies
+   * that the SST filtering service is not started.
+   */
+  @Test
+  public void testSstFilteringDisabledWhenDefragEnabled(@TempDir Path folder) throws Exception {
+    OzoneConfiguration testConf = new OzoneConfiguration();
+    testConf.set(OZONE_METADATA_DIRS, folder.toString());
+    testConf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, TimeUnit.MILLISECONDS);
+    // Enable SST filtering service
+    testConf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS);
+    // Enable defrag service
+    testConf.setTimeDuration(OZONE_SNAPSHOT_DEFRAG_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS);
+    testConf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
+    testConf.setQuietMode(false);
+    // Configure dynamic ports to avoid conflicts with the OzoneManager instance from @BeforeAll
+    testConf.set(OZONE_OM_ADDRESS_KEY, "127.0.0.1:0");
+    testConf.set(OZONE_OM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+    testConf.set(OZONE_OM_HTTPS_ADDRESS_KEY, "127.0.0.1:0");
+    testConf.setInt(OZONE_OM_RATIS_PORT_KEY, 0);
+    testConf.setInt(OZONE_OM_GRPC_PORT_KEY, 0);
+
+    OmTestManagers testManagers = new OmTestManagers(testConf);
+    KeyManager testKeyManager = testManagers.getKeyManager();
+    OzoneManager testOm = testManagers.getOzoneManager();
+
+    try {
+      // Verify that SST filtering service is not started when defrag is enabled
+      SstFilteringService sstFilteringService = testKeyManager.getSnapshotSstFilteringService();
+      assertThat(sstFilteringService).as("SstFilteringService should be null when defrag is enabled").isNull();
+    } finally {
+      if (testKeyManager != null) {
+        testKeyManager.stop();
+      }
+      if (testManagers.getWriteClient() != null) {
+        testManagers.getWriteClient().close();
+      }
+      if (testOm != null) {
+        testOm.stop();
+      }
+    }
+  }
 }