HDDS-14732. Create a new VersionManager for unified component versioning

hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java

@@ -54,7 +54,7 @@
 public class RegisterValidatorProcessor extends AbstractProcessor {
 
   public static final String ANNOTATION_SIMPLE_NAME = "RegisterValidator";
-  public static final String VERSION_CLASS_NAME = "org.apache.hadoop.hdds.ComponentVersion";
+  public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version";
   public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" +
       ".RequestProcessingPhase";
   public static final String APPLY_BEFORE_METHOD_NAME = "applyBefore";

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java

@@ -21,26 +21,61 @@
 import org.apache.hadoop.ozone.upgrade.UpgradeAction;
 
 /**
- * Base type for component version enums.
+ * The logical versioning system used to track incompatible changes to a component, regardless whether they affect disk
+ * or network compatibility between the same or different types of components.
+ *
+ * This interface is the base type for component version enums.
  */
 public interface ComponentVersion {
   /**
-   * @return The serialized representation of this version. This is an opaque value which should not be checked or
-   * compared directly.
+   * Returns an integer representation of this version. To callers outside this class, this is an opaque value which
+   * should not be checked or compared directly. {@link #isSupportedBy} should be used for version comparisons.
+   *
+   * To implementors of this interface, versions should serialize such that
+   * {@code version1 <= version2} if and only if
+   * {@code version1.serialize() <= version2.serialize()}.
+   * Negative numbers may be used as serialized values to represent unknown future versions which are trivially larger
+   * than all other versions.
+   *
+   * @return The serialized representation of this version.
    */
   int serialize();
 
   /**
-   * @return the description of the version enum value.
+   * @return The description of this version.
    */
   String description();
 
   /**
-   * Deserializes a ComponentVersion and checks if its feature set is supported by the current ComponentVersion.
+   * @return The next version immediately following this one, or null if there is no such version.
+   */
+  ComponentVersion nextVersion();
+
+  /**
+   * Uses the serialized representation of a ComponentVersion to check if its feature set is supported by the current
+   * ComponentVersion.
    *
-   * @return true if this version supports the features of otherVersion. False otherwise.
+   * @return true if this version supports the features of the provided version. False otherwise.
    */
-  boolean isSupportedBy(int serializedVersion);
+  default boolean isSupportedBy(int serializedVersion) {
+    if (serialize() < 0) {
+      // Our version is an unknown future version, it is not supported by any other version.
+      return false;
+    } else if (serializedVersion < 0) {
+      // The other version is an unknown future version, it trivially supports all other versions.
+      return true;
+    } else {
+      // If both versions have positive values, they represent concrete versions and we can compare them directly.
+      return serialize() <= serializedVersion;
+    }
+  }
+
+  /**
+   * @return true if this version supports the features of the provided version. False otherwise.
+   */
+  default boolean isSupportedBy(ComponentVersion other) {
+    return isSupportedBy(other.serialize());
+  }
 
   default Optional<? extends UpgradeAction> action() {
     return Optional.empty();

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java

@@ -21,13 +21,16 @@
 import static java.util.stream.Collectors.toMap;
 
 import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 /**
  * Versioning for datanode.
  */
 public enum HDDSVersion implements ComponentVersion {
 
+  //////////////////////////////  //////////////////////////////
+
   DEFAULT_VERSION(0, "Initial version"),
 
   SEPARATE_RATIS_PORTS_AVAILABLE(1, "Version with separated Ratis port."),
@@ -36,14 +39,18 @@ public enum HDDSVersion implements ComponentVersion {
   STREAM_BLOCK_SUPPORT(3,
       "This version has support for reading a block by streaming chunks."),
 
+  ZDU(100, "Version that supports zero downtime upgrade"),
+
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");
 
-  public static final HDDSVersion SOFTWARE_VERSION = latest();
+  //////////////////////////////  //////////////////////////////
 
-  private static final Map<Integer, HDDSVersion> BY_VALUE =
+  private static final SortedMap<Integer, HDDSVersion> BY_VALUE =
       Arrays.stream(values())
-          .collect(toMap(HDDSVersion::serialize, identity()));
+          .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+  public static final HDDSVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey());
 
   private final int version;
   private final String description;
@@ -58,31 +65,35 @@ public String description() {
     return description;
   }
 
+  /**
+   * @return The next version immediately following this one and excluding FUTURE_VERSION,
+   *    or null if there is no such version.
+   */
+  @Override
+  public HDDSVersion nextVersion() {
+    int nextOrdinal = ordinal() + 1;
+    if (nextOrdinal >= values().length - 1) {
+      return null;
+    }
+    return values()[nextOrdinal];
+  }
+
   @Override
   public int serialize() {
     return version;
   }
 
+  /**
+   * @param value The serialized version to convert.
+   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   *    found.
+   */
   public static HDDSVersion deserialize(int value) {
     return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
   }
 
-  @Override
-  public boolean isSupportedBy(int serializedVersion) {
-    // In order for the other serialized version to support this version's features,
-    // the other version must be equal or larger to this version.
-    return deserialize(serializedVersion).compareTo(this) >= 0;
-  }
-
   @Override
   public String toString() {
     return name() + " (" + serialize() + ")";
   }
-
-  private static HDDSVersion latest() {
-    HDDSVersion[] versions = HDDSVersion.values();
-    // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest
-    // version in the software.
-    return versions[versions.length - 2];
-  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java

@@ -17,11 +17,21 @@
 
 package org.apache.hadoop.hdds.upgrade;
 
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+
+import java.util.Arrays;
 import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.hadoop.hdds.ComponentVersion;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.ozone.upgrade.LayoutFeature;
 
 /**
- * List of HDDS Features.
+ * List of HDDS Layout Features. All version management has been migrated to {@link HDDSVersion} and no new additions
+ * should be made to this class. Existing versions are kept here for backwards compatibility when upgrading to this
+ * version from older versions.
  */
 public enum HDDSLayoutFeature implements LayoutFeature {
   //////////////////////////////  //////////////////////////////
@@ -44,8 +54,14 @@ public enum HDDSLayoutFeature implements LayoutFeature {
   WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value type as proto"),
   STORAGE_SPACE_DISTRIBUTION(10, "Enhanced block deletion function for storage space distribution feature.");
 
+  // ALL NEW VERSIONS SHOULD NOW BE ADDED TO HDDSVersion
+
   //////////////////////////////  //////////////////////////////
 
+  private static final SortedMap<Integer, HDDSLayoutFeature> BY_VALUE =
+      Arrays.stream(values())
+          .collect(toMap(HDDSLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
   private final int layoutVersion;
   private final String description;
   private HDDSUpgradeAction scmAction;
@@ -90,6 +106,30 @@ public String description() {
     return description;
   }
 
+  /**
+   * @return The next version immediately following this one. If there is no next version found in this enum,
+   *    the next version is {@link HDDSVersion#ZDU}, since all HDDS versioning has been migrated to
+   *    {@link HDDSVersion} as part of the ZDU feature.
+   */
+  @Override
+  public ComponentVersion nextVersion() {
+    HDDSLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1);
+    if (nextFeature == null) {
+      return HDDSVersion.ZDU;
+    } else {
+      return nextFeature;
+    }
+  }
+
+  /**
+   * @param version The serialized version to convert.
+   * @return The version corresponding to this serialized value, or {@code null} if no matching version is
+   *    found.
+   */
+  public static HDDSLayoutFeature deserialize(int version) {
+    return BY_VALUE.get(version);
+  }
+
   @Override
   public String toString() {
     return name() + " (" + serialize() + ")";

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java

@@ -21,7 +21,8 @@
 import static java.util.stream.Collectors.toMap;
 
 import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import org.apache.hadoop.hdds.ComponentVersion;
 
 /**
@@ -44,11 +45,11 @@ public enum ClientVersion implements ComponentVersion {
   FUTURE_VERSION(-1, "Used internally when the server side is older and an"
       + " unknown client version has arrived from the client.");
 
-  public static final ClientVersion CURRENT = latest();
-
-  private static final Map<Integer, ClientVersion> BY_VALUE =
+  private static final SortedMap<Integer, ClientVersion> BY_VALUE =
       Arrays.stream(values())
-          .collect(toMap(ClientVersion::serialize, identity()));
+          .collect(toMap(ClientVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+  public static final ClientVersion CURRENT = BY_VALUE.get(BY_VALUE.lastKey());
 
   private final int version;
   private final String description;
@@ -63,6 +64,15 @@ public String description() {
     return description;
   }
 
+  @Override
+  public ClientVersion nextVersion() {
+    int nextOrdinal = ordinal() + 1;
+    if (nextOrdinal >= values().length - 1) {
+      return null;
+    }
+    return values()[nextOrdinal];
+  }
+
   @Override
   public int serialize() {
     return version;
@@ -72,22 +82,8 @@ public static ClientVersion deserialize(int value) {
     return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
   }
 
-  @Override
-  public boolean isSupportedBy(int serializedVersion) {
-    // In order for the other serialized version to support this version's features,
-    // the other version must be equal or larger to this version.
-    return deserialize(serializedVersion).compareTo(this) >= 0;
-  }
-
   @Override
   public String toString() {
     return name() + " (" + serialize() + ")";
   }
-
-  private static ClientVersion latest() {
-    ClientVersion[] versions = ClientVersion.values();
-    // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest
-    // version in the software.
-    return versions[versions.length - 2];
-  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java

@@ -21,13 +21,17 @@
 import static java.util.stream.Collectors.toMap;
 
 import java.util.Arrays;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import org.apache.hadoop.hdds.ComponentVersion;
 
 /**
  * Versioning for Ozone Manager.
  */
 public enum OzoneManagerVersion implements ComponentVersion {
+
+  //////////////////////////////  //////////////////////////////
+
   DEFAULT_VERSION(0, "Initial version"),
   S3G_PERSISTENT_CONNECTIONS(1,
       "New S3G persistent connection support is present in OM."),
@@ -54,15 +58,19 @@ public enum OzoneManagerVersion implements ComponentVersion {
 
   S3_LIST_MULTIPART_UPLOADS_PAGINATION(11,
       "OzoneManager version that supports S3 list multipart uploads API with pagination"),
-
+
+  ZDU(100, "OzoneManager version that supports zero downtime upgrade"),
+
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");
 
-  public static final OzoneManagerVersion SOFTWARE_VERSION = latest();
+  //////////////////////////////  //////////////////////////////
 
-  private static final Map<Integer, OzoneManagerVersion> BY_VALUE =
+  private static final SortedMap<Integer, OzoneManagerVersion> BY_VALUE =
       Arrays.stream(values())
-          .collect(toMap(OzoneManagerVersion::serialize, identity()));
+          .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new));
+
+  public static final OzoneManagerVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey());
 
   private final int version;
   private final String description;
@@ -82,26 +90,31 @@ public int serialize() {
     return version;
   }
 
+  /**
+   * @param value The serialized version to convert.
+   * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is
+   *    found.
+   */
   public static OzoneManagerVersion deserialize(int value) {
     return BY_VALUE.getOrDefault(value, FUTURE_VERSION);
   }
 
+
+  /**
+   * @return The next version immediately following this one and excluding FUTURE_VERSION,
+   *    or null if there is no such version.
+   */
   @Override
-  public boolean isSupportedBy(int serializedVersion) {
-    // In order for the other serialized version to support this version's features,
-    // the other version must be equal or larger to this version.
-    return deserialize(serializedVersion).compareTo(this) >= 0;
+  public OzoneManagerVersion nextVersion() {
+    int nextOrdinal = ordinal() + 1;
+    if (nextOrdinal >= values().length - 1) {
+      return null;
+    }
+    return values()[nextOrdinal];
   }
 
   @Override
   public String toString() {
     return name() + " (" + serialize() + ")";
   }
-
-  private static OzoneManagerVersion latest() {
-    OzoneManagerVersion[] versions = OzoneManagerVersion.values();
-    // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest
-    // version in the software.
-    return versions[versions.length - 2];
-  }
 }