From 074232db47290f44de2312d9bfdbeb2399c30727 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 26 Feb 2026 17:36:25 -0500 Subject: [PATCH 01/20] Add new version manager classes (cherry picked from commit 89fc4ca3e7493a01ce311e6736d79ab4699f4cf5) (cherry picked from commit 9bf5c8ac6333b0d7e45e12ff537fd1a975b103e7) --- .../ozone/upgrade/AbstractVersionManager.java | 38 +++++++++++++++++++ .../hadoop/ozone/upgrade/VersionManager.java | 15 ++++++++ 2 files changed, 53 insertions(+) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java new file mode 100644 index 00000000000..84e732c92b2 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java @@ -0,0 +1,38 @@ +package org.apache.hadoop.ozone.upgrade; + +import org.apache.hadoop.hdds.ComponentVersion; + +public class AbstractVersionManager implements VersionManager { + private final ComponentVersion apparentVersion; + private final ComponentVersion softwareVersion; + + public AbstractVersionManager(ComponentVersion apparentVersion, ComponentVersion softwareVersion) { + this.apparentVersion = apparentVersion; + this.softwareVersion = softwareVersion; + } + + @Override + public ComponentVersion getApparentVersion() { + return apparentVersion; + } + + @Override + public ComponentVersion getSoftwareVersion() { + return softwareVersion; + } + + @Override + public boolean isAllowed(ComponentVersion version) { + return + } + + @Override + public Iterable getUnfinalizedVersions() { + return null; + } + + @Override + public void close() { + + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java new file mode 100644 index 00000000000..de4a69a4b7b --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java @@ -0,0 +1,15 @@ +package org.apache.hadoop.ozone.upgrade; + +import org.apache.hadoop.hdds.ComponentVersion; + +public interface VersionManager { + ComponentVersion getApparentVersion(); + + ComponentVersion getSoftwareVersion(); + + boolean isAllowed(ComponentVersion version); + + Iterable getUnfinalizedVersions(); + + void close(); +} From 327d1fce3ce52a8e250ab979b01dbe06fa77797c Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 5 Mar 2026 18:07:17 -0500 Subject: [PATCH 02/20] Initial implementation of new version manager with version updates --- .../RegisterValidatorProcessor.java | 9 +- .../apache/hadoop/hdds/ComponentVersion.java | 24 ++++- .../org/apache/hadoop/hdds/HDDSVersion.java | 33 ++++--- .../hdds/upgrade/HDDSLayoutFeature.java | 24 ++++- .../apache/hadoop/ozone/ClientVersion.java | 30 +++--- .../hadoop/ozone/OzoneManagerVersion.java | 45 ++++++--- .../hadoop/ozone/upgrade/LayoutFeature.java | 2 +- .../ozone/upgrade/AbstractVersionManager.java | 38 -------- .../upgrade/ComponentVersionManager.java | 96 +++++++++++++++++++ .../ComponentVersionManagerMetrics.java | 77 +++++++++++++++ .../hadoop/ozone/upgrade/VersionManager.java | 43 ++++++++- .../upgrade/TestUpgradeFinalizerActions.java | 18 +++- .../hadoop/ozone/debug/VersionDebug.java | 3 +- .../ozone/om/upgrade/OMLayoutFeature.java | 23 ++++- 14 files changed, 376 insertions(+), 89 deletions(-) delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index 6b26043efb2..3e5456f9b3a 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -29,6 +29,7 @@ import javax.lang.model.element.TypeElement; import javax.lang.model.type.ArrayType; import javax.lang.model.type.TypeKind; +import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import javax.lang.model.util.Types; import javax.tools.Diagnostic; @@ -86,11 +87,13 @@ private boolean validateArrayMethod(ExecutableElement method, String expectedMet Types types = processingEnv.getTypeUtils(); TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : elementUtils.getTypeElement(expectedReturnClass); + TypeMirror expectedType = expectedReturnInterface == null + ? null : types.erasure(expectedReturnInterface.asType()); return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || TypeKind.ARRAY.equals(method.getReturnType().getKind()) && types.asElement(((ArrayType)method.getReturnType()).getComponentType()).getKind() == expectedReturnType) && (expectedReturnInterface == null || - types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); + types.isAssignable(types.erasure(types.asElement(method.getReturnType()).asType()), expectedType)); } private boolean validateMethod(ExecutableElement method, String expectedMethodName, ElementKind expectedReturnType, @@ -99,11 +102,13 @@ private boolean validateMethod(ExecutableElement method, String expectedMethodNa Types types = processingEnv.getTypeUtils(); TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : elementUtils.getTypeElement(expectedReturnClass); + TypeMirror expectedType = expectedReturnInterface == null + ? null : types.erasure(expectedReturnInterface.asType()); return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || types.asElement(method.getReturnType()) != null && types.asElement(method.getReturnType()).getKind() == expectedReturnType) && (expectedReturnInterface == null || - types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); + types.isAssignable(types.erasure(types.asElement(method.getReturnType()).asType()), expectedType)); } private void processElements(Set annotatedElements) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 10f232ee0c9..4975d5021a4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -17,13 +17,23 @@ package org.apache.hadoop.hdds; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.upgrade.UpgradeAction; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + /** * Base type for component version enums. */ -public interface ComponentVersion { +public interface ComponentVersion> extends Comparable { /** * @return The serialized representation of this version. This is an opaque value which should not be checked or * compared directly. @@ -31,10 +41,20 @@ public interface ComponentVersion { int serialize(); /** - * @return the description of the version enum value. + * @return The description of the version enum value. */ String description(); + /** + * @return The next version immediately following this one, or null if there is no such version. + */ + V nextVersion(); + + /** + * @return All versions immediately following this one, in order, or an empty iterable if there are no more versions. + */ + Iterable nextVersions(); + /** * Deserializes a ComponentVersion and checks if its feature set is supported by the current ComponentVersion. * diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index 6b4131e2226..b6b87c0c515 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -17,16 +17,20 @@ package org.apache.hadoop.hdds; +import org.apache.hadoop.ozone.OzoneManagerVersion; + import static java.util.function.Function.identity; import static java.util.stream.Collectors.toMap; import java.util.Arrays; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; /** * Versioning for datanode. */ -public enum HDDSVersion implements ComponentVersion { +public enum HDDSVersion implements ComponentVersion { DEFAULT_VERSION(0, "Initial version"), @@ -36,14 +40,16 @@ public enum HDDSVersion implements ComponentVersion { STREAM_BLOCK_SUPPORT(3, "This version has support for reading a block by streaming chunks."), + ZDU(100, "Version that supports zero downtime upgrade"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); - public static final HDDSVersion SOFTWARE_VERSION = latest(); - - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(HDDSVersion::serialize, identity())); + .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final HDDSVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -58,6 +64,16 @@ public String description() { return description; } + @Override + public HDDSVersion nextVersion() { + return BY_VALUE.get(version + 1); + } + + @Override + public Iterable nextVersions() { + return BY_VALUE.tailMap(version + 1).values(); + } + @Override public int serialize() { return version; @@ -78,11 +94,4 @@ public boolean isSupportedBy(int serializedVersion) { public String toString() { return name() + " (" + serialize() + ")"; } - - private static HDDSVersion latest() { - HDDSVersion[] versions = HDDSVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index e78e1dcbffe..6f2cf328ee9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -17,13 +17,21 @@ package org.apache.hadoop.hdds.upgrade; +import java.util.Arrays; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.ozone.upgrade.LayoutFeature; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + /** * List of HDDS Features. */ -public enum HDDSLayoutFeature implements LayoutFeature { +public enum HDDSLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), DATANODE_SCHEMA_V2(1, "Datanode RocksDB Schema Version 2 (with column " + @@ -46,6 +54,10 @@ public enum HDDSLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// + private static final SortedMap BY_VALUE = + Arrays.stream(values()) + .collect(toMap(HDDSLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + private final int layoutVersion; private final String description; private HDDSUpgradeAction scmAction; @@ -90,6 +102,16 @@ public String description() { return description; } + @Override + public HDDSLayoutFeature nextVersion() { + return BY_VALUE.get(layoutVersion + 1); + } + + @Override + public Iterable nextVersions() { + return BY_VALUE.tailMap(layoutVersion + 1).values(); + } + @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index ac20b5fe3b7..281e2058fc0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -22,12 +22,15 @@ import java.util.Arrays; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; /** * Versioning for protocol clients. */ -public enum ClientVersion implements ComponentVersion { +public enum ClientVersion implements ComponentVersion { DEFAULT_VERSION(0, "Initial version"), @@ -44,11 +47,11 @@ public enum ClientVersion implements ComponentVersion { FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); - public static final ClientVersion CURRENT = latest(); - - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(ClientVersion::serialize, identity())); + .collect(toMap(ClientVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final ClientVersion CURRENT = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -63,6 +66,16 @@ public String description() { return description; } + @Override + public ClientVersion nextVersion() { + return BY_VALUE.get(version + 1); + } + + @Override + public Iterable nextVersions() { + return BY_VALUE.tailMap(version + 1).values(); + } + @Override public int serialize() { return version; @@ -83,11 +96,4 @@ public boolean isSupportedBy(int serializedVersion) { public String toString() { return name() + " (" + serialize() + ")"; } - - private static ClientVersion latest() { - ClientVersion[] versions = ClientVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 9ec8870855e..5ef62959b28 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -21,13 +21,18 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; +import java.util.Iterator; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.jspecify.annotations.NonNull; /** * Versioning for Ozone Manager. */ -public enum OzoneManagerVersion implements ComponentVersion { +public enum OzoneManagerVersion implements ComponentVersion { DEFAULT_VERSION(0, "Initial version"), S3G_PERSISTENT_CONNECTIONS(1, "New S3G persistent connection support is present in OM."), @@ -54,15 +59,17 @@ public enum OzoneManagerVersion implements ComponentVersion { S3_LIST_MULTIPART_UPLOADS_PAGINATION(11, "OzoneManager version that supports S3 list multipart uploads API with pagination"), - + + ZDU(100, "OzoneManager version that supports zero downtime upgrade"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); - public static final OzoneManagerVersion SOFTWARE_VERSION = latest(); - - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(OzoneManagerVersion::serialize, identity())); + .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final OzoneManagerVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -86,6 +93,25 @@ public static OzoneManagerVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } + + /** + * @return The next version immediately following this one and excluding FUTURE_VERSION, + * or null if there is no such version. + */ + @Override + public OzoneManagerVersion nextVersion() { + return BY_VALUE.get(version + 1); + } + + /** + * @return All versions immediately following this one in order but excluding FUTURE_VERSION, + * or an empty iterable if there are no more versions. + */ + @Override + public Iterable nextVersions() { + return BY_VALUE.tailMap(version + 1).values(); + } + @Override public boolean isSupportedBy(int serializedVersion) { // In order for the other serialized version to support this version's features, @@ -97,11 +123,4 @@ public boolean isSupportedBy(int serializedVersion) { public String toString() { return name() + " (" + serialize() + ")"; } - - private static OzoneManagerVersion latest() { - OzoneManagerVersion[] versions = OzoneManagerVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 848a35104e5..6f816e6a225 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -22,7 +22,7 @@ /** * Generic Layout feature interface for Ozone. */ -public interface LayoutFeature extends ComponentVersion { +public interface LayoutFeature> extends ComponentVersion { int layoutVersion(); @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java deleted file mode 100644 index 84e732c92b2..00000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractVersionManager.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.apache.hadoop.ozone.upgrade; - -import org.apache.hadoop.hdds.ComponentVersion; - -public class AbstractVersionManager implements VersionManager { - private final ComponentVersion apparentVersion; - private final ComponentVersion softwareVersion; - - public AbstractVersionManager(ComponentVersion apparentVersion, ComponentVersion softwareVersion) { - this.apparentVersion = apparentVersion; - this.softwareVersion = softwareVersion; - } - - @Override - public ComponentVersion getApparentVersion() { - return apparentVersion; - } - - @Override - public ComponentVersion getSoftwareVersion() { - return softwareVersion; - } - - @Override - public boolean isAllowed(ComponentVersion version) { - return - } - - @Override - public Iterable getUnfinalizedVersions() { - return null; - } - - @Override - public void close() { - - } -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java new file mode 100644 index 00000000000..9b2e0068fe1 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java @@ -0,0 +1,96 @@ +package org.apache.hadoop.ozone.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks information about the apparent version, software version, and finalization status of a component. + */ +public class ComponentVersionManager> implements VersionManager { + // Apparent version may be updated during the finalization process. + private volatile V apparentVersion; + // Software version will never change. + private final V softwareVersion; + private final ComponentVersionManagerMetrics metrics; + + private static final Logger LOG = + LoggerFactory.getLogger(ComponentVersionManager.class); + + public ComponentVersionManager(V apparentVersion, V softwareVersion) + throws IOException { + this.apparentVersion = apparentVersion; + this.softwareVersion = softwareVersion; + + if (apparentVersion.compareTo(softwareVersion) > 0) { + throw new IOException( + "Cannot initialize ComponentVersionManager. Apparent version " + + apparentVersion + " is larger than software version " + + softwareVersion); + } + + LOG.info("Initializing version manager with apparent version {} and software version {}", + apparentVersion, softwareVersion); + this.metrics = ComponentVersionManagerMetrics.create(this); + } + + @Override + public V getApparentVersion() { + return apparentVersion; + } + + @Override + public V getSoftwareVersion() { + return softwareVersion; + } + + @Override + public boolean isAllowed(V version) { + return version.compareTo(apparentVersion) <= 0; + } + + @Override + public boolean needsFinalization() { + return apparentVersion.compareTo(softwareVersion) < 0; + } + + @Override + public Iterable getUnfinalizedVersions() { + return apparentVersion.nextVersions(); + } + + @Override + public void markFinalized(V newApparentVersion) { + String versionMsg = "Software version: " + softwareVersion + + ", apparent version: " + apparentVersion + + ", provided version: " + newApparentVersion + + "."; + + if (newApparentVersion.compareTo(apparentVersion) <= 0) { + LOG.info("Finalize attempt on a version which has already " + + "been finalized. {} This can happen when " + + "Raft Log is replayed during service restart.", versionMsg); + } else { + // newApparentVersion has already been validated to be larger than + // apparentVersion. + // Therefore, nextVersion will not return null. + if (apparentVersion.nextVersion().equals(newApparentVersion)) { + apparentVersion = newApparentVersion; + LOG.info("Version {} has been finalized.", apparentVersion); + if (!needsFinalization()) { + LOG.info("Finalization is complete."); + } + } else { + throw new IllegalArgumentException( + "Finalize attempt on a version that is newer than the " + + "next feature to be finalized. " + versionMsg); + } + } + } + + @Override + public void close() { + metrics.unRegister(); + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java new file mode 100644 index 00000000000..51bed7c10d4 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.ozone.OzoneConsts; + +/** + * Metrics for {@link ComponentVersionManager}. + */ +@Metrics(about = "Component Version Manager Metrics", context = OzoneConsts.OZONE) +public final class ComponentVersionManagerMetrics implements MetricsSource { + + public static final String METRICS_SOURCE_NAME = + ComponentVersionManagerMetrics.class.getSimpleName(); + + private static final MetricsInfo SOFTWARE_VERSION = Interns.info( + "SoftwareVersion", + "Software version in serialized int form."); + private static final MetricsInfo APPARENT_VERSION = Interns.info( + "ApparentVersion", + "Current apparent version in serialized int form."); + + private final VersionManager versionManager; + + private ComponentVersionManagerMetrics(VersionManager versionManager) { + this.versionManager = versionManager; + } + + public static ComponentVersionManagerMetrics create(VersionManager versionManager) { + ComponentVersionManagerMetrics metrics = (ComponentVersionManagerMetrics) DefaultMetricsSystem.instance() + .getSource(METRICS_SOURCE_NAME); + if (metrics == null) { + return DefaultMetricsSystem.instance().register( + METRICS_SOURCE_NAME, + "Metrics for component version management.", + new ComponentVersionManagerMetrics(versionManager)); + } + return metrics; + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + MetricsRecordBuilder builder = collector.addRecord(METRICS_SOURCE_NAME); + builder + .addGauge(SOFTWARE_VERSION, + versionManager.getSoftwareVersion().serialize()) + .addGauge(APPARENT_VERSION, + versionManager.getApparentVersion().serialize()); + } + + public void unRegister() { + DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME); + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java index de4a69a4b7b..784af7a592c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java @@ -1,15 +1,48 @@ package org.apache.hadoop.ozone.upgrade; +import java.io.IOException; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; -public interface VersionManager { - ComponentVersion getApparentVersion(); +/** + * Tracks information about the apparent version, software version, and finalization status of a component. + * This interface also provides factory methods to create implementations based on component and apparent version. + * When migrating from the old LayoutFeature based versioning to the new unified ComponentVersion in the ZDU + * software version, a different implementation will be required. + */ +public interface VersionManager> { + V getApparentVersion(); - ComponentVersion getSoftwareVersion(); + V getSoftwareVersion(); - boolean isAllowed(ComponentVersion version); + boolean isAllowed(V version); - Iterable getUnfinalizedVersions(); + boolean needsFinalization(); + + Iterable getUnfinalizedVersions(); + + void markFinalized(V version); void close(); + + static ComponentVersionManager newForOM(int serializedApparentVersion) throws IOException { + if (OzoneManagerVersion.ZDU.isSupportedBy(serializedApparentVersion)) { + return new ComponentVersionManager<>(OzoneManagerVersion.deserialize(serializedApparentVersion), + OzoneManagerVersion.SOFTWARE_VERSION); + } else { + // TODO return migration manager + return null; + } + } + + static ComponentVersionManager newForHDDS(int serializedApparentVersion) throws IOException { + if (HDDSVersion.ZDU.isSupportedBy(serializedApparentVersion)) { + return new ComponentVersionManager<>(HDDSVersion.deserialize(serializedApparentVersion), + HDDSVersion.SOFTWARE_VERSION); + } else { + // TODO return migration manager + return null; + } + } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java index ade86b488b9..1919183a352 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.upgrade; import java.io.IOException; +import java.util.Arrays; import java.util.Optional; import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction; import org.apache.hadoop.hdds.upgrade.test.MockComponent; @@ -64,7 +65,7 @@ static class MockLayoutVersionManager extends /** * Mock Layout Feature list. */ - enum MockLayoutFeature implements LayoutFeature { + enum MockLayoutFeature implements LayoutFeature { VERSION_1(1), VERSION_2(2), VERSION_3(3); @@ -86,6 +87,21 @@ public String description() { return null; } + @Override + public MockLayoutFeature nextVersion() { + int nextOrdinal = ordinal() + 1; + return nextOrdinal < values().length ? values()[nextOrdinal] : null; + } + + @Override + public Iterable nextVersions() { + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length) { + return java.util.Collections.emptyList(); + } + return Arrays.asList(values()).subList(nextOrdinal, values().length); + } + @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java index 87cf936deb3..e8c68dfd7ce 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java @@ -59,7 +59,8 @@ public Void call() throws IOException { return null; } - private static & ComponentVersion> Map asMap(T version) { + private static & ComponentVersion> + Map asMap(T version) { return ImmutableSortedMap.of( "componentVersion", ImmutableSortedMap.of( "name", version.name(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index c19b720682d..066fdfa6f04 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -17,13 +17,20 @@ package org.apache.hadoop.ozone.om.upgrade; +import java.util.Arrays; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.ozone.upgrade.LayoutFeature; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + /** * List of OM Layout features / versions. */ -public enum OMLayoutFeature implements LayoutFeature { +public enum OMLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), @@ -48,6 +55,10 @@ public enum OMLayoutFeature implements LayoutFeature { /////////////////////////////// ///////////////////////////// + private static final SortedMap BY_VALUE = + Arrays.stream(values()) + .collect(toMap(OMLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + private final int layoutVersion; private final String description; private OmUpgradeAction action; @@ -84,6 +95,16 @@ public void addAction(OmUpgradeAction upgradeAction) { } } + @Override + public OMLayoutFeature nextVersion() { + return BY_VALUE.get(layoutVersion + 1); + } + + @Override + public Iterable nextVersions() { + return BY_VALUE.tailMap(layoutVersion + 1).values(); + } + @Override public Optional action() { return Optional.ofNullable(action); From 4a8cff71ca16d4d9e2c2d7c58f65261de58ab00b Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 10 Mar 2026 17:28:32 -0400 Subject: [PATCH 03/20] Initial compiling version manager changes --- .../apache/hadoop/hdds/ComponentVersion.java | 33 +++--- .../org/apache/hadoop/hdds/HDDSVersion.java | 23 ++-- .../hdds/upgrade/HDDSLayoutFeature.java | 43 +++++-- .../apache/hadoop/ozone/ClientVersion.java | 9 +- .../hadoop/ozone/OzoneManagerVersion.java | 25 ++-- .../hadoop/ozone/upgrade/LayoutFeature.java | 10 +- .../hdds/upgrade/HDDSVersionManager.java | 40 +++++++ .../upgrade/ComponentVersionManager.java | 111 +++++++++++++----- .../ComponentVersionManagerMetrics.java | 7 +- .../hadoop/ozone/upgrade/VersionManager.java | 48 -------- .../hadoop/ozone/debug/VersionDebug.java | 2 +- .../ozone/om/upgrade/OMLayoutFeature.java | 52 ++++++-- .../ozone/om/upgrade/OMVersionManager.java | 40 +++++++ 13 files changed, 281 insertions(+), 162 deletions(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java delete mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 4975d5021a4..3bc212272a3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -17,23 +17,16 @@ package org.apache.hadoop.hdds; -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; -import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.upgrade.UpgradeAction; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; - /** - * Base type for component version enums. + * The logical versioning system used to track incompatible changes to a component, regardless whether they affect disk + * or network compatibility between the same or different types of components. + * + * This interface is the base type for component version enums. */ -public interface ComponentVersion> extends Comparable { +public interface ComponentVersion { /** * @return The serialized representation of this version. This is an opaque value which should not be checked or * compared directly. @@ -48,20 +41,22 @@ public interface ComponentVersion> extends Compara /** * @return The next version immediately following this one, or null if there is no such version. */ - V nextVersion(); - - /** - * @return All versions immediately following this one, in order, or an empty iterable if there are no more versions. - */ - Iterable nextVersions(); + ComponentVersion nextVersion(); /** * Deserializes a ComponentVersion and checks if its feature set is supported by the current ComponentVersion. * - * @return true if this version supports the features of otherVersion. False otherwise. + * @return true if this version supports the features of the provided version. False otherwise. */ boolean isSupportedBy(int serializedVersion); + /** + * @return true if this version supports the features of the provided version. False otherwise. + */ + default boolean isSupportedBy(ComponentVersion other) { + return isSupportedBy(other.serialize()); + } + default Optional action() { return Optional.empty(); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index b6b87c0c515..51aa2d9fab9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -17,20 +17,19 @@ package org.apache.hadoop.hdds; -import org.apache.hadoop.ozone.OzoneManagerVersion; - import static java.util.function.Function.identity; import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; /** * Versioning for datanode. */ -public enum HDDSVersion implements ComponentVersion { +public enum HDDSVersion implements ComponentVersion { + + ////////////////////////////// ////////////////////////////// DEFAULT_VERSION(0, "Initial version"), @@ -45,6 +44,8 @@ public enum HDDSVersion implements ComponentVersion { FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); + ////////////////////////////// ////////////////////////////// + private static final SortedMap BY_VALUE = Arrays.stream(values()) .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); @@ -64,21 +65,25 @@ public String description() { return description; } + /** + * @return The next version immediately following this one and excluding FUTURE_VERSION, + * or null if there is no such version. + */ @Override public HDDSVersion nextVersion() { return BY_VALUE.get(version + 1); } - @Override - public Iterable nextVersions() { - return BY_VALUE.tailMap(version + 1).values(); - } - @Override public int serialize() { return version; } + /** + * @param value The serialized version to convert. + * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * found. + */ public static HDDSVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index 6f2cf328ee9..34a424154a8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -17,6 +17,9 @@ package org.apache.hadoop.hdds.upgrade; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + import java.util.Arrays; import java.util.Optional; import java.util.SortedMap; @@ -25,13 +28,12 @@ import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.ozone.upgrade.LayoutFeature; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; - /** - * List of HDDS Features. + * List of HDDS Layout Features. All version management has been migrated to {@link HDDSVersion} and no new additions + * should be made to this class. Existing versions are kept here for backwards compatibility when upgrading to this + * version from older versions. */ -public enum HDDSLayoutFeature implements LayoutFeature { +public enum HDDSLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), DATANODE_SCHEMA_V2(1, "Datanode RocksDB Schema Version 2 (with column " + @@ -52,6 +54,8 @@ public enum HDDSLayoutFeature implements LayoutFeature { WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value type as proto"), STORAGE_SPACE_DISTRIBUTION(10, "Enhanced block deletion function for storage space distribution feature."); + // ALL NEW VERSIONS SHOULD NOW BE ADDED TO HDDSVersion + ////////////////////////////// ////////////////////////////// private static final SortedMap BY_VALUE = @@ -102,14 +106,35 @@ public String description() { return description; } + /** + * @return The next version immediately following this one. If there is no next version found in this enum, + * the next version is {@link HDDSVersion#ZDU}, since all HDDS versioning has been migrated to + * {@link HDDSVersion} as part of the ZDU feature. + */ @Override - public HDDSLayoutFeature nextVersion() { - return BY_VALUE.get(layoutVersion + 1); + public ComponentVersion nextVersion() { + HDDSLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1); + if (nextFeature == null) { + return HDDSVersion.ZDU; + } else { + return nextFeature; + } } @Override - public Iterable nextVersions() { - return BY_VALUE.tailMap(layoutVersion + 1).values(); + public boolean isSupportedBy(int serializedVersion) { + // In order for the other serialized version to support this version's features, + // the other version must be equal or larger to this version. + return serializedVersion >= layoutVersion(); + } + + /** + * @param version The serialized version to convert. + * @return The version corresponding to this serialized value, or {@code null} if no matching version is + * found. + */ + public static HDDSLayoutFeature deserialize(int version) { + return BY_VALUE.get(version); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index 281e2058fc0..ac7a1dbba99 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -21,16 +21,14 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; -import org.apache.hadoop.hdds.HDDSVersion; /** * Versioning for protocol clients. */ -public enum ClientVersion implements ComponentVersion { +public enum ClientVersion implements ComponentVersion { DEFAULT_VERSION(0, "Initial version"), @@ -71,11 +69,6 @@ public ClientVersion nextVersion() { return BY_VALUE.get(version + 1); } - @Override - public Iterable nextVersions() { - return BY_VALUE.tailMap(version + 1).values(); - } - @Override public int serialize() { return version; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 5ef62959b28..aa9dab36f09 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -21,18 +21,17 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; -import org.apache.hadoop.hdds.HDDSVersion; -import org.jspecify.annotations.NonNull; /** * Versioning for Ozone Manager. */ -public enum OzoneManagerVersion implements ComponentVersion { +public enum OzoneManagerVersion implements ComponentVersion { + + ////////////////////////////// ////////////////////////////// + DEFAULT_VERSION(0, "Initial version"), S3G_PERSISTENT_CONNECTIONS(1, "New S3G persistent connection support is present in OM."), @@ -65,6 +64,8 @@ public enum OzoneManagerVersion implements ComponentVersion FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); + ////////////////////////////// ////////////////////////////// + private static final SortedMap BY_VALUE = Arrays.stream(values()) .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); @@ -89,6 +90,11 @@ public int serialize() { return version; } + /** + * @param value The serialized version to convert. + * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * found. + */ public static OzoneManagerVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } @@ -103,15 +109,6 @@ public OzoneManagerVersion nextVersion() { return BY_VALUE.get(version + 1); } - /** - * @return All versions immediately following this one in order but excluding FUTURE_VERSION, - * or an empty iterable if there are no more versions. - */ - @Override - public Iterable nextVersions() { - return BY_VALUE.tailMap(version + 1).values(); - } - @Override public boolean isSupportedBy(int serializedVersion) { // In order for the other serialized version to support this version's features, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 6f816e6a225..7f7374c8137 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -22,19 +22,11 @@ /** * Generic Layout feature interface for Ozone. */ -public interface LayoutFeature> extends ComponentVersion { +public interface LayoutFeature extends ComponentVersion { int layoutVersion(); @Override default int serialize() { return this.layoutVersion(); } - - @Override - default boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - // We can compare the values directly since there is no FUTURE_VERSION for layout features. - return serializedVersion >= layoutVersion(); - } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java new file mode 100644 index 00000000000..c158fc05237 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; + +/** + * Component version manager for HDDS. + */ +public class HDDSVersionManager extends ComponentVersionManager { + public HDDSVersionManager(int serializedApparentVersion) throws IOException { + super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION); + } + + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { + if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) { + return HDDSLayoutFeature.deserialize(serializedApparentVersion); + } else { + return HDDSVersion.deserialize(serializedApparentVersion); + } + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java index 9b2e0068fe1..11bf4a3f228 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java @@ -1,29 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.upgrade; +import java.io.Closeable; import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; import org.apache.hadoop.hdds.ComponentVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Tracks information about the apparent version, software version, and finalization status of a component. + * + * Software version: The {@link ComponentVersion} of the code that is currently running. + * This is always the highest component version within the code and does not change while the process is running. + * + * Apparent version: The {@link ComponentVersion} the software is acting as, which is persisted to the disk. + * The apparent version determines the API that is exposed by the component and the format it uses to persist data. + * Using an apparent version less than software version allows us to support rolling upgrades and downgrades. + * + * Pre-finalized: State a component enters when the apparent version on disk is less than the software version. + * At this time all other machines may or may not be running the new bits, new features are blocked, and downgrade + * is allowed. + * + * Finalized: State a component enters when the apparent version is equal to the software version. + * A component transitions from pre-finalized to finalized when it receives a finalize command from the + * admin. At this time all machines are running the new bits, and even though this component is finalized, + * different types of components may not be. Downgrade is not allowed after this point. + * */ -public class ComponentVersionManager> implements VersionManager { +public abstract class ComponentVersionManager implements Closeable { // Apparent version may be updated during the finalization process. - private volatile V apparentVersion; + private volatile ComponentVersion apparentVersion; // Software version will never change. - private final V softwareVersion; + private final ComponentVersion softwareVersion; private final ComponentVersionManagerMetrics metrics; private static final Logger LOG = LoggerFactory.getLogger(ComponentVersionManager.class); - public ComponentVersionManager(V apparentVersion, V softwareVersion) + protected ComponentVersionManager(ComponentVersion apparentVersion, ComponentVersion softwareVersion) throws IOException { this.apparentVersion = apparentVersion; this.softwareVersion = softwareVersion; - if (apparentVersion.compareTo(softwareVersion) > 0) { + if (!apparentVersion.isSupportedBy(softwareVersion)) { throw new IOException( "Cannot initialize ComponentVersionManager. Apparent version " + apparentVersion + " is larger than software version " @@ -35,47 +72,66 @@ public ComponentVersionManager(V apparentVersion, V softwareVersion) this.metrics = ComponentVersionManagerMetrics.create(this); } - @Override - public V getApparentVersion() { + public ComponentVersion getApparentVersion() { return apparentVersion; } - @Override - public V getSoftwareVersion() { + public ComponentVersion getSoftwareVersion() { return softwareVersion; } - @Override - public boolean isAllowed(V version) { - return version.compareTo(apparentVersion) <= 0; + public boolean isAllowed(ComponentVersion version) { + return version.isSupportedBy(apparentVersion); } - @Override public boolean needsFinalization() { - return apparentVersion.compareTo(softwareVersion) < 0; + return !apparentVersion.equals(softwareVersion); } - @Override - public Iterable getUnfinalizedVersions() { - return apparentVersion.nextVersions(); + /** + * @return An Iterable of all versions after the current apparent version which still need to be finalized. If this + * component is already finalized, the Iterable will be empty. + */ + public Iterable getUnfinalizedVersions() { + return () -> new Iterator() { + private ComponentVersion currentVersion = apparentVersion; + + @Override + public boolean hasNext() { + return currentVersion.nextVersion() != null; + } + + @Override + public ComponentVersion next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + currentVersion = currentVersion.nextVersion(); + return currentVersion; + } + }; } - @Override - public void markFinalized(V newApparentVersion) { + /** + * Validates that the provided version is valid to finalize to, and if so, updates the in-memory apparent version to + * this version. Also logs corresponding messages about finalization status. + * + * @param newApparentVersion The version to mark as finalized. + */ + public void markFinalized(ComponentVersion newApparentVersion) { String versionMsg = "Software version: " + softwareVersion + ", apparent version: " + apparentVersion + ", provided version: " + newApparentVersion + "."; - if (newApparentVersion.compareTo(apparentVersion) <= 0) { - LOG.info("Finalize attempt on a version which has already " - + "been finalized. {} This can happen when " + + if (newApparentVersion.isSupportedBy(apparentVersion)) { + LOG.info("Finalize attempt on a version which has already been finalized. {} This can happen when " + "Raft Log is replayed during service restart.", versionMsg); } else { - // newApparentVersion has already been validated to be larger than - // apparentVersion. - // Therefore, nextVersion will not return null. - if (apparentVersion.nextVersion().equals(newApparentVersion)) { + ComponentVersion nextVersion = apparentVersion.nextVersion(); + if (nextVersion == null) { + throw new IllegalArgumentException("Attempt to finalize when no future versions exist." + versionMsg); + } else if (nextVersion.equals(newApparentVersion)) { apparentVersion = newApparentVersion; LOG.info("Version {} has been finalized.", apparentVersion); if (!needsFinalization()) { @@ -83,8 +139,7 @@ public void markFinalized(V newApparentVersion) { } } else { throw new IllegalArgumentException( - "Finalize attempt on a version that is newer than the " + - "next feature to be finalized. " + versionMsg); + "Finalize attempt on a version that is newer than the next feature to be finalized. " + versionMsg); } } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java index 51bed7c10d4..3cafd99d48e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.upgrade; -import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -43,13 +42,13 @@ public final class ComponentVersionManagerMetrics implements MetricsSource { "ApparentVersion", "Current apparent version in serialized int form."); - private final VersionManager versionManager; + private final ComponentVersionManager versionManager; - private ComponentVersionManagerMetrics(VersionManager versionManager) { + private ComponentVersionManagerMetrics(ComponentVersionManager versionManager) { this.versionManager = versionManager; } - public static ComponentVersionManagerMetrics create(VersionManager versionManager) { + public static ComponentVersionManagerMetrics create(ComponentVersionManager versionManager) { ComponentVersionManagerMetrics metrics = (ComponentVersionManagerMetrics) DefaultMetricsSystem.instance() .getSource(METRICS_SOURCE_NAME); if (metrics == null) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java deleted file mode 100644 index 784af7a592c..00000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/VersionManager.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.hadoop.ozone.upgrade; - -import java.io.IOException; -import org.apache.hadoop.hdds.ComponentVersion; -import org.apache.hadoop.hdds.HDDSVersion; -import org.apache.hadoop.ozone.OzoneManagerVersion; - -/** - * Tracks information about the apparent version, software version, and finalization status of a component. - * This interface also provides factory methods to create implementations based on component and apparent version. - * When migrating from the old LayoutFeature based versioning to the new unified ComponentVersion in the ZDU - * software version, a different implementation will be required. - */ -public interface VersionManager> { - V getApparentVersion(); - - V getSoftwareVersion(); - - boolean isAllowed(V version); - - boolean needsFinalization(); - - Iterable getUnfinalizedVersions(); - - void markFinalized(V version); - - void close(); - - static ComponentVersionManager newForOM(int serializedApparentVersion) throws IOException { - if (OzoneManagerVersion.ZDU.isSupportedBy(serializedApparentVersion)) { - return new ComponentVersionManager<>(OzoneManagerVersion.deserialize(serializedApparentVersion), - OzoneManagerVersion.SOFTWARE_VERSION); - } else { - // TODO return migration manager - return null; - } - } - - static ComponentVersionManager newForHDDS(int serializedApparentVersion) throws IOException { - if (HDDSVersion.ZDU.isSupportedBy(serializedApparentVersion)) { - return new ComponentVersionManager<>(HDDSVersion.deserialize(serializedApparentVersion), - HDDSVersion.SOFTWARE_VERSION); - } else { - // TODO return migration manager - return null; - } - } -} diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java index e8c68dfd7ce..f9daf5afcb5 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java @@ -59,7 +59,7 @@ public Void call() throws IOException { return null; } - private static & ComponentVersion> + private static & ComponentVersion> Map asMap(T version) { return ImmutableSortedMap.of( "componentVersion", ImmutableSortedMap.of( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 066fdfa6f04..26057c557ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -17,20 +17,23 @@ package org.apache.hadoop.ozone.om.upgrade; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + import java.util.Arrays; import java.util.Optional; import java.util.SortedMap; import java.util.TreeMap; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.upgrade.LayoutFeature; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; - /** - * List of OM Layout features / versions. + * List of OM Layout Features. All version management has been migrated to {@link OzoneManagerVersion} and no new + * additions should be made to this class. Existing versions are kept here for backwards compatibility when upgrading + * to this version from older versions. */ -public enum OMLayoutFeature implements LayoutFeature { +public enum OMLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), @@ -53,6 +56,8 @@ public enum OMLayoutFeature implements LayoutFeature { DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"), SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"); + // ALL NEW VERSIONS SHOULD NOW BE ADDED TO OzoneManagerVersion + /////////////////////////////// ///////////////////////////// private static final SortedMap BY_VALUE = @@ -73,6 +78,22 @@ public int layoutVersion() { return layoutVersion; } + @Override + public boolean isSupportedBy(int serializedVersion) { + // In order for the other serialized version to support this version's features, + // the other version must be equal or larger to this version. + return serializedVersion >= layoutVersion(); + } + + /** + * @param version The serialized version to convert. + * @return The version corresponding to this serialized value, or {@code null} if no matching version is + * found. + */ + public static OMLayoutFeature deserialize(int version) { + return BY_VALUE.get(version); + } + @Override public String description() { return description; @@ -95,14 +116,19 @@ public void addAction(OmUpgradeAction upgradeAction) { } } + /** + * @return The next version immediately following this one. If there is no next version found in this enum, + * the next version is {@link OzoneManagerVersion#ZDU}, since all OM versioning has been migrated to + * {@link OzoneManagerVersion} as part of the ZDU feature. + */ @Override - public OMLayoutFeature nextVersion() { - return BY_VALUE.get(layoutVersion + 1); - } - - @Override - public Iterable nextVersions() { - return BY_VALUE.tailMap(layoutVersion + 1).values(); + public ComponentVersion nextVersion() { + OMLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1); + if (nextFeature == null) { + return OzoneManagerVersion.ZDU; + } else { + return nextFeature; + } } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java new file mode 100644 index 00000000000..e3d64f29b01 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; + +/** + * Component version manager for Ozone Manager. + */ +public class OMVersionManager extends ComponentVersionManager { + public OMVersionManager(int serializedApparentVersion) throws IOException { + super(computeApparentVersion(serializedApparentVersion), OzoneManagerVersion.SOFTWARE_VERSION); + } + + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { + if (serializedApparentVersion < OzoneManagerVersion.ZDU.serialize()) { + return OMLayoutFeature.deserialize(serializedApparentVersion); + } else { + return OzoneManagerVersion.deserialize(serializedApparentVersion); + } + } +} From 6942451832fbe7477767bf449bc3bc07832ccfd5 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 17:07:01 -0400 Subject: [PATCH 04/20] Update javadoc --- .../apache/hadoop/hdds/upgrade/HDDSVersionManager.java | 8 +++++++- .../apache/hadoop/ozone/om/upgrade/OMVersionManager.java | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java index c158fc05237..6b622cd9b80 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -23,13 +23,19 @@ import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; /** - * Component version manager for HDDS. + * Component version manager for HDDS (Datanodes and SCM). */ public class HDDSVersionManager extends ComponentVersionManager { public HDDSVersionManager(int serializedApparentVersion) throws IOException { super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION); } + /** + * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link HDDSVersion}. + * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link HDDSLayoutFeature}. + */ private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) { return HDDSLayoutFeature.deserialize(serializedApparentVersion); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java index e3d64f29b01..3a6831bdcc8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -30,6 +30,12 @@ public OMVersionManager(int serializedApparentVersion) throws IOException { super(computeApparentVersion(serializedApparentVersion), OzoneManagerVersion.SOFTWARE_VERSION); } + /** + * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link OzoneManagerVersion}. + * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link OMLayoutFeature}. + */ private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { if (serializedApparentVersion < OzoneManagerVersion.ZDU.serialize()) { return OMLayoutFeature.deserialize(serializedApparentVersion); From 1095b5b67ca182a3d9c2dd730cade96f22711c72 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 17:33:58 -0400 Subject: [PATCH 05/20] Now it builds --- .../upgrade/TestAbstractLayoutVersionManager.java | 13 +++++++++++++ .../ozone/upgrade/TestUpgradeFinalizerActions.java | 12 ++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java index f006698518a..9761eaf8c73 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -29,6 +29,7 @@ import java.util.Iterator; import javax.management.MBeanServer; import javax.management.ObjectName; +import org.apache.hadoop.hdds.ComponentVersion; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -188,6 +189,18 @@ public int layoutVersion() { public String description() { return null; } + + @Override + public ComponentVersion nextVersion() { + // TODO HDDS-14826 will remove this test. No need to add handling for this new method. + return null; + } + + @Override + public boolean isSupportedBy(int serializedVersion) { + // TODO HDDS-14826 will remove this test. No need to add handling for this new method. + return false; + } }; } return lfs; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java index 1919183a352..bee8134f799 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.upgrade; import java.io.IOException; -import java.util.Arrays; import java.util.Optional; import org.apache.hadoop.hdds.upgrade.HDDSUpgradeAction; import org.apache.hadoop.hdds.upgrade.test.MockComponent; @@ -65,7 +64,7 @@ static class MockLayoutVersionManager extends /** * Mock Layout Feature list. */ - enum MockLayoutFeature implements LayoutFeature { + enum MockLayoutFeature implements LayoutFeature { VERSION_1(1), VERSION_2(2), VERSION_3(3); @@ -94,12 +93,9 @@ public MockLayoutFeature nextVersion() { } @Override - public Iterable nextVersions() { - int nextOrdinal = ordinal() + 1; - if (nextOrdinal >= values().length) { - return java.util.Collections.emptyList(); - } - return Arrays.asList(values()).subList(nextOrdinal, values().length); + public boolean isSupportedBy(int serializedVersion) { + // TODO HDDS-14826 will remove this test. No need to add handling for this new method. + return false; } @Override From e86d166775b4ae5b0e8204e5dc48e4314cf6b014 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 18:15:15 -0400 Subject: [PATCH 06/20] Add tests that no new layout features are added --- .../hdds/upgrade/TestHDDSLayoutFeature.java | 50 +++++++++++++++ .../hdds/upgrade/HDDSVersionManager.java | 2 + .../upgrade/TestHDDSLayoutVersionManager.java | 10 --- .../ozone/om/upgrade/OMVersionManager.java | 2 + .../ozone/om/upgrade/TestOMLayoutFeature.java | 64 +++++++++++++++++++ ...r.java => TestOMLayoutVersionManager.java} | 22 +------ 6 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/{TestOMVersionManager.java => TestOMLayoutVersionManager.java} (87%) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java new file mode 100644 index 00000000000..8c284950840 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.upgrade; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.hdds.HDDSVersion; +import org.junit.jupiter.api.Test; + +/** + * Tests invariants for legacy HDDS layout feature versions. + */ +public class TestHDDSLayoutFeature { + @Test + public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() { + HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); + int currVersion = -1; + for (HDDSLayoutFeature lf : values) { + assertEquals(currVersion + 1, lf.layoutVersion()); + currVersion = lf.layoutVersion(); + } + } + + /** + * All incompatible changes to HDDS (SCM and Datanodes) should now be added to {@link HDDSVersion}. + */ + @Test + public void testNoNewHDDSLayoutFeaturesAdded() { + int numHDDSLayoutFeatures = HDDSLayoutFeature.values().length; + HDDSLayoutFeature lastFeature = HDDSLayoutFeature.values()[numHDDSLayoutFeatures - 1]; + assertEquals(11, numHDDSLayoutFeatures); + assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION, lastFeature); + assertEquals(10, lastFeature.layoutVersion()); + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java index 6b622cd9b80..ad6bad1c702 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -43,4 +43,6 @@ private static ComponentVersion computeApparentVersion(int serializedApparentVer return HDDSVersion.deserialize(serializedApparentVersion); } } + + // TODO HDDS-14826: Register upgrade actions based on annotations } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java index 4792e1179de..37d3b88eeb1 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java @@ -82,14 +82,4 @@ public void testUpgradeActionsRegistered() throws Exception { verify(mockObj, times(0)).mockMethodScm(); verify(mockObj, times(1)).mockMethodDn(); } - - @Test - public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() { - HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); - int currVersion = -1; - for (HDDSLayoutFeature lf : values) { - assertEquals(currVersion + 1, lf.layoutVersion()); - currVersion = lf.layoutVersion(); - } - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java index 3a6831bdcc8..f250928b2a9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -43,4 +43,6 @@ private static ComponentVersion computeApparentVersion(int serializedApparentVer return OzoneManagerVersion.deserialize(serializedApparentVersion); } } + + // TODO HDDS-14826: Register upgrade actions based on annotations } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java new file mode 100644 index 00000000000..521a198079e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.junit.jupiter.api.Test; + +/** + * Tests invariants for legacy OM layout feature versions. + */ +public class TestOMLayoutFeature { + @Test + public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() + throws Exception { + OMLayoutFeature[] values = OMLayoutFeature.values(); + int currVersion = -1; + OMLayoutFeature lastFeature = null; + for (OMLayoutFeature lf : values) { + assertEquals(currVersion + 1, lf.layoutVersion()); + currVersion = lf.layoutVersion(); + lastFeature = lf; + } + lastFeature.addAction(arg -> { + String v = arg.getVersion(); + }); + + OzoneManager omMock = mock(OzoneManager.class); + lastFeature.action().get().execute(omMock); + verify(omMock, times(1)).getVersion(); + } + + /** + * All incompatible changes to OM should now be added to {@link OzoneManagerVersion}. + */ + @Test + public void testNoNewOMLayoutFeaturesAdded() { + int numOMLayoutFeatures = OMLayoutFeature.values().length; + OMLayoutFeature lastFeature = OMLayoutFeature.values()[numOMLayoutFeatures - 1]; + assertEquals(10, numOMLayoutFeatures); + assertEquals(OMLayoutFeature.SNAPSHOT_DEFRAG, lastFeature); + assertEquals(9, lastFeature.layoutVersion()); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java similarity index 87% rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java index 388b5134c41..2c2044e9d04 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java @@ -44,7 +44,7 @@ /** * Test OM layout version management. */ -public class TestOMVersionManager { +public class TestOMLayoutVersionManager { @Test public void testOMLayoutVersionManager() throws IOException { @@ -65,26 +65,6 @@ public void testOMLayoutVersionManagerInitError() { assertEquals(NOT_SUPPORTED_OPERATION, ome.getResult()); } - @Test - public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() - throws Exception { - OMLayoutFeature[] values = OMLayoutFeature.values(); - int currVersion = -1; - OMLayoutFeature lastFeature = null; - for (OMLayoutFeature lf : values) { - assertEquals(currVersion + 1, lf.layoutVersion()); - currVersion = lf.layoutVersion(); - lastFeature = lf; - } - lastFeature.addAction(arg -> { - String v = arg.getVersion(); - }); - - OzoneManager omMock = mock(OzoneManager.class); - lastFeature.action().get().execute(omMock); - verify(omMock, times(1)).getVersion(); - } - @Test /* From 2522edfa7ebe364806bc9f06a26efc3f2897e8ef Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 19:13:39 -0400 Subject: [PATCH 07/20] Layout features are supported by FUTURE_VERSION --- .../org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +++- .../org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index 34a424154a8..b5853c33220 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -125,7 +125,9 @@ public ComponentVersion nextVersion() { public boolean isSupportedBy(int serializedVersion) { // In order for the other serialized version to support this version's features, // the other version must be equal or larger to this version. - return serializedVersion >= layoutVersion(); + return serializedVersion >= layoutVersion() || + // Future versions support all known features. + HDDSVersion.deserialize(serializedVersion).equals(HDDSVersion.FUTURE_VERSION); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 26057c557ec..cd0bb242460 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -82,7 +82,9 @@ public int layoutVersion() { public boolean isSupportedBy(int serializedVersion) { // In order for the other serialized version to support this version's features, // the other version must be equal or larger to this version. - return serializedVersion >= layoutVersion(); + return serializedVersion >= layoutVersion() || + // Future versions support all known features. + OzoneManagerVersion.deserialize(serializedVersion).equals(OzoneManagerVersion.FUTURE_VERSION); } /** From d0c840e11a00830fd8ce2ec84aa757f7f08a4c51 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 19:58:40 -0400 Subject: [PATCH 08/20] Add unit tests for layout features --- .../hdds/ComponentVersionTestUtils.java | 51 ++++++++++++ .../hdds/TestComponentVersionInvariants.java | 18 ++--- .../hdds/upgrade/TestHDDSLayoutFeature.java | 61 ++++++++++++++- .../ozone/om/upgrade/TestOMLayoutFeature.java | 77 +++++++++++++++---- 4 files changed, 179 insertions(+), 28 deletions(-) create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java new file mode 100644 index 00000000000..0531523cfac --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Shared assertions for {@link ComponentVersion} tests. + */ +public final class ComponentVersionTestUtils { + + private ComponentVersionTestUtils() { } + + public static void assertSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider) { + assertSupportedBy(requiredFeature, provider, true); + } + + public static void assertNotSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider) { + assertSupportedBy(requiredFeature, provider, false); + } + + /** + * Helper method to test support by passing both serialized and deserialized versions. + */ + private static void assertSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider, boolean expected) { + int providerSerializedVersion = provider.serialize(); + assertEquals(expected, requiredFeature.isSupportedBy(providerSerializedVersion), + "Expected support check via serialized overload to match for version " + + providerSerializedVersion); + assertEquals(expected, requiredFeature.isSupportedBy(provider), + "Expected support check via version overload to match for " + provider); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java index 0edab7e76d2..d271a2e1c89 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java @@ -17,6 +17,8 @@ package org.apache.hadoop.hdds; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -91,15 +93,6 @@ public void testSerializedValuesAreMonotonic(ComponentVersion[] values, Componen assertEquals(values.length, ++startValue); } - @ParameterizedTest - @MethodSource("values") - public void testVersionIsSupportedByItself(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - for (ComponentVersion value : values) { - assertTrue(value.isSupportedBy(value.serialize())); - } - } - @ParameterizedTest @MethodSource("values") public void testOnlyEqualOrHigherVersionsCanSupportAFeature(ComponentVersion[] values, ComponentVersion defaultValue, @@ -109,8 +102,11 @@ public void testOnlyEqualOrHigherVersionsCanSupportAFeature(ComponentVersion[] v ComponentVersion requiredFeature = values[featureIndex]; for (int providerIndex = 0; providerIndex < knownVersionCount; providerIndex++) { ComponentVersion provider = values[providerIndex]; - boolean expected = providerIndex >= featureIndex; - assertEquals(expected, requiredFeature.isSupportedBy(provider.serialize())); + if (providerIndex >= featureIndex) { + assertSupportedBy(requiredFeature, provider); + } else { + assertNotSupportedBy(requiredFeature, provider); + } } } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java index 8c284950840..fbb89220e97 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java @@ -17,7 +17,11 @@ package org.apache.hadoop.hdds.upgrade; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.hdds.HDDSVersion; import org.junit.jupiter.api.Test; @@ -31,7 +35,10 @@ public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() { HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); int currVersion = -1; for (HDDSLayoutFeature lf : values) { - assertEquals(currVersion + 1, lf.layoutVersion()); + // This will skip the jump from the last HDDSLayoutFeature to HDDSVersion#ZDU, + // since that is expected to be a larger version increment. + assertEquals(currVersion + 1, lf.layoutVersion(), + "Expected monotonically increasing layout version for " + lf); currVersion = lf.layoutVersion(); } } @@ -47,4 +54,56 @@ public void testNoNewHDDSLayoutFeaturesAdded() { assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION, lastFeature); assertEquals(10, lastFeature.layoutVersion()); } + + @Test + public void testNextVersion() { + HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); + for (int i = 1; i < values.length; i++) { + HDDSLayoutFeature previous = values[i - 1]; + HDDSLayoutFeature current = values[i]; + assertEquals(current, previous.nextVersion(), + "Expected " + previous + ".nextVersion() to be " + current); + } + // The last layout feature should point us to the ZDU version to switch to using HDDSVersion. + assertEquals(HDDSVersion.ZDU, values[values.length - 1].nextVersion()); + } + + @Test + public void testSerDes() { + for (HDDSLayoutFeature version : HDDSLayoutFeature.values()) { + assertEquals(version, HDDSLayoutFeature.deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownVersionReturnsNull() { + assertNull(HDDSLayoutFeature.deserialize(-1)); + assertNull(HDDSLayoutFeature.deserialize(Integer.MAX_VALUE)); + // HDDSLayoutFeature can only deserialize values from its own enum. + assertNull(HDDSLayoutFeature.deserialize(HDDSVersion.ZDU.serialize())); + } + + @Test + public void testIsSupportedByFeatureBoundary() { + for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { + // A layout feature should support itself. + int layoutVersion = feature.layoutVersion(); + assertSupportedBy(feature, feature); + if (layoutVersion > 0) { + // A layout feature should not be supported by older features. + HDDSLayoutFeature previousFeature = HDDSLayoutFeature.values()[layoutVersion - 1]; + assertNotSupportedBy(feature, previousFeature); + } + } + } + + @Test + public void testAllLayoutFeaturesAreSupportedByFutureVersions() { + for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { + assertSupportedBy(feature, HDDSVersion.ZDU); + assertSupportedBy(feature, HDDSVersion.FUTURE_VERSION); + // No ComponentVersion instance represents an arbitrary future version. + assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java index 521a198079e..b22bd1f5ac8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -17,13 +17,13 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.apache.hadoop.ozone.om.OzoneManager; import org.junit.jupiter.api.Test; /** @@ -31,23 +31,16 @@ */ public class TestOMLayoutFeature { @Test - public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() - throws Exception { + public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() { OMLayoutFeature[] values = OMLayoutFeature.values(); int currVersion = -1; - OMLayoutFeature lastFeature = null; for (OMLayoutFeature lf : values) { - assertEquals(currVersion + 1, lf.layoutVersion()); + // This will skip the jump from the last OMLayoutFeature to OzoneManagerVersion#ZDU, + // since that is expected to be a larger version increment. + assertEquals(currVersion + 1, lf.layoutVersion(), + "Expected monotonically increasing layout version for " + lf); currVersion = lf.layoutVersion(); - lastFeature = lf; } - lastFeature.addAction(arg -> { - String v = arg.getVersion(); - }); - - OzoneManager omMock = mock(OzoneManager.class); - lastFeature.action().get().execute(omMock); - verify(omMock, times(1)).getVersion(); } /** @@ -61,4 +54,56 @@ public void testNoNewOMLayoutFeaturesAdded() { assertEquals(OMLayoutFeature.SNAPSHOT_DEFRAG, lastFeature); assertEquals(9, lastFeature.layoutVersion()); } + + @Test + public void testNextVersion() { + OMLayoutFeature[] values = OMLayoutFeature.values(); + for (int i = 1; i < values.length; i++) { + OMLayoutFeature previous = values[i - 1]; + OMLayoutFeature current = values[i]; + assertEquals(current, previous.nextVersion(), + "Expected " + previous + ".nextVersion() to be " + current); + } + // The last layout feature should point us to the ZDU version to switch to using OzoneManagerVersion. + assertEquals(OzoneManagerVersion.ZDU, values[values.length - 1].nextVersion()); + } + + @Test + public void testSerDes() { + for (OMLayoutFeature version : OMLayoutFeature.values()) { + assertEquals(version, OMLayoutFeature.deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownVersionReturnsNull() { + assertNull(OMLayoutFeature.deserialize(-1)); + assertNull(OMLayoutFeature.deserialize(Integer.MAX_VALUE)); + // OMLayoutFeature can only deserialize values from its own enum. + assertNull(OMLayoutFeature.deserialize(OzoneManagerVersion.ZDU.serialize())); + } + + @Test + public void testIsSupportedByFeatureBoundary() { + for (OMLayoutFeature feature : OMLayoutFeature.values()) { + // A layout feature should support itself. + int layoutVersion = feature.layoutVersion(); + assertSupportedBy(feature, feature); + if (layoutVersion > 0) { + // A layout feature should not be supported by older features. + OMLayoutFeature previousFeature = OMLayoutFeature.values()[layoutVersion - 1]; + assertNotSupportedBy(feature, previousFeature); + } + } + } + + @Test + public void testAllLayoutFeaturesAreSupportedByFutureVersions() { + for (OMLayoutFeature feature : OMLayoutFeature.values()) { + assertSupportedBy(feature, OzoneManagerVersion.ZDU); + assertSupportedBy(feature, OzoneManagerVersion.FUTURE_VERSION); + // No ComponentVersion instance represents an arbitrary unknown future version. + assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); + } + } } From a6214c8dff4d1ebe3daee446abc4af93ab7df1e7 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 11 Mar 2026 20:59:33 -0400 Subject: [PATCH 09/20] Add unit tests for component version changes --- .../org/apache/hadoop/hdds/HDDSVersion.java | 6 +- .../apache/hadoop/ozone/ClientVersion.java | 6 +- .../hadoop/ozone/OzoneManagerVersion.java | 6 +- .../hdds/AbstractComponentVersionTest.java | 126 +++++++++++++++ .../apache/hadoop/hdds/TestClientVersion.java | 45 ++++++ .../hdds/TestComponentVersionInvariants.java | 144 ------------------ .../apache/hadoop/hdds/TestHDDSVersion.java | 63 ++++++++ .../hadoop/hdds/TestOzoneManagerVersion.java | 64 ++++++++ 8 files changed, 313 insertions(+), 147 deletions(-) create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index 51aa2d9fab9..c877bcfe217 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -71,7 +71,11 @@ public String description() { */ @Override public HDDSVersion nextVersion() { - return BY_VALUE.get(version + 1); + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index ac7a1dbba99..cf1059334bb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -66,7 +66,11 @@ public String description() { @Override public ClientVersion nextVersion() { - return BY_VALUE.get(version + 1); + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index aa9dab36f09..2cca20ee9a7 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -106,7 +106,11 @@ public static OzoneManagerVersion deserialize(int value) { */ @Override public OzoneManagerVersion nextVersion() { - return BY_VALUE.get(version + 1); + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; } @Override diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java new file mode 100644 index 00000000000..bfa534629a5 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * Shared invariants for component version enums. + */ +public abstract class AbstractComponentVersionTest { + + protected abstract ComponentVersion[] getValues(); + + protected abstract ComponentVersion getDefaultVersion(); + + protected abstract ComponentVersion getFutureVersion(); + + protected abstract ComponentVersion deserialize(int value); + + // FUTURE_VERSION is the latest + @Test + public void testFutureVersionHasTheHighestOrdinal() { + ComponentVersion[] values = getValues(); + ComponentVersion futureValue = getFutureVersion(); + assertEquals(values[values.length - 1], futureValue); + } + + // FUTURE_VERSION's internal version id is -1 + @Test + public void testFutureVersionSerializesToMinusOne() { + ComponentVersion futureValue = getFutureVersion(); + assertEquals(-1, futureValue.serialize()); + } + + // DEFAULT_VERSION's internal version id is 0 + @Test + public void testDefaultVersionSerializesToZero() { + ComponentVersion defaultValue = getDefaultVersion(); + assertEquals(0, defaultValue.serialize()); + } + + // known (non-future) versions are strictly increasing + @Test + public void testSerializedValuesAreMonotonic() { + ComponentVersion[] values = getValues(); + int knownVersionCount = values.length - 1; + for (int i = 1; i < knownVersionCount; i++) { + assertTrue(values[i].serialize() > values[i - 1].serialize(), + "Expected known version serialization to increase: " + values[i - 1] + " -> " + values[i]); + } + } + + @Test + public void testNextVersionProgression() { + ComponentVersion[] values = getValues(); + ComponentVersion futureValue = getFutureVersion(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + assertEquals(values[i + 1], values[i].nextVersion(), + "Expected nextVersion progression for " + values[i]); + } + assertNull(values[knownVersionCount - 1].nextVersion(), + "Expected latest known version to have no nextVersion"); + assertNull(futureValue.nextVersion(), + "Expected FUTURE_VERSION.nextVersion() to return null"); + } + + @Test + public void testOnlyEqualOrHigherVersionsCanSupportAFeature() { + ComponentVersion[] values = getValues(); + int knownVersionCount = values.length - 1; + for (int featureIndex = 0; featureIndex < knownVersionCount; featureIndex++) { + ComponentVersion requiredFeature = values[featureIndex]; + for (int providerIndex = 0; providerIndex < knownVersionCount; providerIndex++) { + ComponentVersion provider = values[providerIndex]; + if (providerIndex >= featureIndex) { + assertSupportedBy(requiredFeature, provider); + } else { + assertNotSupportedBy(requiredFeature, provider); + } + } + } + } + + @Test + public void testFutureVersionSupportsAllKnownVersions() { + ComponentVersion[] values = getValues(); + int unknownFutureVersion = Integer.MAX_VALUE; + for (ComponentVersion requiredFeature : values) { + assertTrue(requiredFeature.isSupportedBy(unknownFutureVersion)); + } + } + + @Test + public void testVersionSerDes() { + for (ComponentVersion version : getValues()) { + assertEquals(version, deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownReturnsFutureVersion() { + assertEquals(getFutureVersion(), deserialize(Integer.MAX_VALUE)); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java new file mode 100644 index 00000000000..563c10b0a39 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; +import org.apache.hadoop.ozone.ClientVersion; + +/** + * Invariants for {@link ClientVersion}. + */ +public class TestClientVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return ClientVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return ClientVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return ClientVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return ClientVersion.deserialize(value); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java deleted file mode 100644 index d271a2e1c89..00000000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds; - -import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; -import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; - -import java.util.stream.Stream; -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -/** - * Test to ensure Component version instances conform with invariants relied - * upon in other parts of the codebase. - */ -public class TestComponentVersionInvariants { - - public static Stream values() { - return Stream.of( - arguments( - HDDSVersion.values(), - HDDSVersion.DEFAULT_VERSION, - HDDSVersion.FUTURE_VERSION), - arguments( - ClientVersion.values(), - ClientVersion.DEFAULT_VERSION, - ClientVersion.FUTURE_VERSION), - arguments( - OzoneManagerVersion.values(), - OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION) - ); - } - - // FUTURE_VERSION is the latest - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionHasTheHighestOrdinal(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - - assertEquals(values[values.length - 1], futureValue); - } - - // FUTURE_VERSION's internal version id is -1 - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionSerializesToMinusOne(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - assertEquals(-1, futureValue.serialize()); - - } - - // DEFAULT_VERSION's internal version id is 0 - @ParameterizedTest - @MethodSource("values") - public void testDefaultVersionSerializesToZero(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - assertEquals(0, defaultValue.serialize()); - } - - // versions are increasing monotonically by one - @ParameterizedTest - @MethodSource("values") - public void testSerializedValuesAreMonotonic(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int startValue = defaultValue.serialize(); - // we skip the future version at the last position - for (int i = 0; i < values.length - 1; i++) { - assertEquals(values[i].serialize(), startValue++); - } - assertEquals(values.length, ++startValue); - } - - @ParameterizedTest - @MethodSource("values") - public void testOnlyEqualOrHigherVersionsCanSupportAFeature(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int knownVersionCount = values.length - 1; - for (int featureIndex = 0; featureIndex < knownVersionCount; featureIndex++) { - ComponentVersion requiredFeature = values[featureIndex]; - for (int providerIndex = 0; providerIndex < knownVersionCount; providerIndex++) { - ComponentVersion provider = values[providerIndex]; - if (providerIndex >= featureIndex) { - assertSupportedBy(requiredFeature, provider); - } else { - assertNotSupportedBy(requiredFeature, provider); - } - } - } - } - - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionSupportsAllKnownVersions(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int unknownFutureVersion = Integer.MAX_VALUE; - for (ComponentVersion requiredFeature : values) { - assertTrue(requiredFeature.isSupportedBy(unknownFutureVersion)); - } - } - - @Test - public void testHDDSVersionSerDes() { - for (HDDSVersion version: HDDSVersion.values()) { - assertEquals(version, HDDSVersion.deserialize(version.serialize())); - } - } - - @Test - public void testOMVersionSerDes() { - for (OzoneManagerVersion version: OzoneManagerVersion.values()) { - assertEquals(version, OzoneManagerVersion.deserialize(version.serialize())); - } - } - - @Test - public void testClientVersionSerDes() { - for (ClientVersion version: ClientVersion.values()) { - assertEquals(version, ClientVersion.deserialize(version.serialize())); - } - } -} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java new file mode 100644 index 00000000000..89d24d3c82e --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Invariants for {@link HDDSVersion}. + */ +public class TestHDDSVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return HDDSVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return HDDSVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return HDDSVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return HDDSVersion.deserialize(value); + } + + @Test + public void testKnownVersionNumbersAreContiguousExceptForZDU() { + HDDSVersion[] values = HDDSVersion.values(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + HDDSVersion current = values[i]; + HDDSVersion next = values[i + 1]; + if (next == HDDSVersion.ZDU) { + assertEquals(100, next.serialize()); + } else { + assertEquals(current.serialize() + 1, next.serialize()); + } + } + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java new file mode 100644 index 00000000000..bb94ac938fd --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.junit.jupiter.api.Test; + +/** + * Invariants for {@link OzoneManagerVersion}. + */ +public class TestOzoneManagerVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return OzoneManagerVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return OzoneManagerVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return OzoneManagerVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return OzoneManagerVersion.deserialize(value); + } + + @Test + public void testKnownVersionNumbersAreContiguousExceptForZDU() { + OzoneManagerVersion[] values = OzoneManagerVersion.values(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + OzoneManagerVersion current = values[i]; + OzoneManagerVersion next = values[i + 1]; + if (next == OzoneManagerVersion.ZDU) { + assertEquals(100, next.serialize()); + } else { + assertEquals(current.serialize() + 1, next.serialize()); + } + } + } +} From d14c375c09a0a679a022fa475c0bfd26aa15f5c5 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 12 Mar 2026 17:00:22 -0400 Subject: [PATCH 10/20] Checkstyle --- .../src/test/java/org/apache/hadoop/hdds/TestClientVersion.java | 1 + .../org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java index 563c10b0a39..721fb2466c9 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hdds; + import org.apache.hadoop.ozone.ClientVersion; /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java index b22bd1f5ac8..9e98935b5b3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -20,8 +20,8 @@ import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.junit.jupiter.api.Test; From 9ad03a472c9f05ceebd784407267d217fccecfa3 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 15:14:13 -0400 Subject: [PATCH 11/20] Add initial passing HDDS version manager tests, and change how isSupportedBy works --- .../apache/hadoop/hdds/ComponentVersion.java | 27 ++- .../org/apache/hadoop/hdds/HDDSVersion.java | 7 - .../hdds/upgrade/HDDSLayoutFeature.java | 31 ++-- .../hdds/upgrade/TestHDDSVersionManager.java | 170 ++++++++++++++++++ .../ozone/om/upgrade/OMLayoutFeature.java | 9 - 5 files changed, 204 insertions(+), 40 deletions(-) create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 3bc212272a3..4bc511e37fc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -28,8 +28,15 @@ */ public interface ComponentVersion { /** - * @return The serialized representation of this version. This is an opaque value which should not be checked or - * compared directly. + * Returns an integer representation of this version. To callers outside this class, this is an opaque value which + * should not be checked or compared directly. + * + * To implementors of this interface, versions should serialize such that if version1 < version2, + * then version1.serialize() < version2.serialize(). + * Negative numbers may be used as serialized values to represent unknown future versions which are trivially larger + * than all other versions. + * + * @return The serialized representation of this version. */ int serialize(); @@ -44,11 +51,23 @@ public interface ComponentVersion { ComponentVersion nextVersion(); /** - * Deserializes a ComponentVersion and checks if its feature set is supported by the current ComponentVersion. + * Uses the serialized representation of a ComponentVersion to check if its feature set is supported by the current + * ComponentVersion. * * @return true if this version supports the features of the provided version. False otherwise. */ - boolean isSupportedBy(int serializedVersion); + default boolean isSupportedBy(int serializedVersion) { + if (serialize() < 0) { + // Our version is an unknown future version, it is not supported by any other version. + return false; + } else if (serializedVersion < 0) { + // The other version is an unknown future version, it trivially supports all other versions. + return true; + } else { + // If both versions have positive values, they represent concrete versions and we can compare them directly. + return serialize() <= serializedVersion; + } + } /** * @return true if this version supports the features of the provided version. False otherwise. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index c877bcfe217..51d4229a748 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -92,13 +92,6 @@ public static HDDSVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index b5853c33220..dbca2bfe9ac 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -1,18 +1,18 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * licensed to the apache software foundation (asf) under one or more + * contributor license agreements. see the notice file distributed with * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the asf licenses this file to you under the apache license, version 2.0 + * (the "license"); you may not use this file except in compliance with + * the license. you may obtain a copy of the license at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/license-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * unless required by applicable law or agreed to in writing, software + * distributed under the license is distributed on an "as is" basis, + * without warranties or conditions of any kind, either express or implied. + * see the license for the specific language governing permissions and + * limitations under the license. */ package org.apache.hadoop.hdds.upgrade; @@ -121,15 +121,6 @@ public ComponentVersion nextVersion() { } } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return serializedVersion >= layoutVersion() || - // Future versions support all known features. - HDDSVersion.deserialize(serializedVersion).equals(HDDSVersion.FUTURE_VERSION); - } - /** * @param version The serialized version to convert. * @return The version corresponding to this serialized value, or {@code null} if no matching version is diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java new file mode 100644 index 00000000000..e6ac237300b --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java @@ -0,0 +1,170 @@ +/* + * licensed to the apache software foundation (asf) under one or more + * contributor license agreements. see the notice file distributed with + * this work for additional information regarding copyright ownership. + * the asf licenses this file to you under the apache license, version 2.0 + * (the "license"); you may not use this file except in compliance with + * the license. you may obtain a copy of the license at + * + * http://www.apache.org/licenses/license-2.0 + * + * unless required by applicable law or agreed to in writing, software + * distributed under the license is distributed on an "as is" basis, + * without warranties or conditions of any kind, either express or implied. + * see the license for the specific language governing permissions and + * limitations under the license. + */ + +package org.apache.hadoop.hdds.upgrade; + +import static org.apache.ozone.test.MetricsAsserts.assertGauge; +import static org.apache.ozone.test.MetricsAsserts.getMetrics; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManagerMetrics; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; + +class TestHDDSVersionManager { + + private static final List ALL_VERSIONS; + + static { + ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values())); + + for (HDDSVersion version : HDDSVersion.values()) { + // Add all defined versions after and including ZDU to get the complete version list. + if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) { + ALL_VERSIONS.add(version); + } + } + } + + /** + * @return All version less than our current software version that we can use as initial apparent versions to test + * finalizing to the latest software version. + */ + private static Stream preFinalizedVersionArgs() { + return ALL_VERSIONS.stream() + .limit(ALL_VERSIONS.size() - 1) + .map(Arguments::of); + } + + @AfterEach + public void cleanupMetricsSource() { + DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + } + + @Test + public void testApparentVersionTranslation() throws Exception { + for (ComponentVersion apparentVersion : ALL_VERSIONS) { + HDDSVersionManager versionManager = new HDDSVersionManager(apparentVersion.serialize()); + assertApparentVersion(versionManager, apparentVersion); + // Unregister the metrics for this object. A new one will be created on the next loop iteration. + DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + } + } + + @Test + public void testApparentVersionBehindSoftwareVersion() { + int serializedNextVersion = HDDSVersion.SOFTWARE_VERSION.serialize() + 1; + assertThrows(IOException.class, () -> new HDDSVersionManager(serializedNextVersion)); + } + + @ParameterizedTest + @MethodSource("preFinalizedVersionArgs") + public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) throws Exception { + int apparentVersionIndex = ALL_VERSIONS.indexOf(apparentVersion); + assertTrue(apparentVersionIndex >= 0, "apparentVersion must exist in preFinalizedVersions"); + Iterator expectedVersions = ALL_VERSIONS + .subList(apparentVersionIndex + 1, ALL_VERSIONS.size()).iterator(); + + HDDSVersionManager versionManager = new HDDSVersionManager(apparentVersion.serialize()); + assertApparentVersion(versionManager, apparentVersion); + + for (ComponentVersion versionToFinalize : versionManager.getUnfinalizedVersions()) { + assertTrue(versionManager.needsFinalization()); + assertFalse(versionManager.isAllowed(versionToFinalize), "Unfinalized version " + versionToFinalize + + " should not be allowed by apparent version " + versionManager.getApparentVersion()); + // Ensure versions are iterated in the expected order + assertTrue(expectedVersions.hasNext()); + assertEquals(expectedVersions.next(), versionToFinalize); + + versionManager.markFinalized(versionToFinalize); + + // This version should now be finalized. + assertApparentVersion(versionManager, versionToFinalize); + } + + assertFalse(expectedVersions.hasNext()); + assertThrows(NoSuchElementException.class, expectedVersions::next); + } + + @Test + public void testFinalizationFromSoftwareVersionNoOp() throws Exception { + HDDSVersionManager versionManager = new HDDSVersionManager(HDDSVersion.SOFTWARE_VERSION.serialize()); + + assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + // Duplicate finalize call should not throw or change state. + versionManager.markFinalized(HDDSVersion.SOFTWARE_VERSION); + + assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + + @Test + public void testFinalizationOfNonExistentVersion() throws Exception { + HDDSVersionManager versionManager = new HDDSVersionManager(HDDSVersion.SOFTWARE_VERSION.serialize()); + assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + // Create a mock version which appears newer than all existing versions, including the software version. + ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class); + when(mockVersion.isSupportedBy(any())).thenReturn(false); + + // Attempting to finalize this version should throw without changing state. + assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); + assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + + private static void assertApparentVersion(HDDSVersionManager versionManager, ComponentVersion apparentVersion) { + assertEquals(apparentVersion, versionManager.getApparentVersion()); + assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + " should be allowed"); + assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getSoftwareVersion(), + "Software version should never change"); + if (!versionManager.needsFinalization()) { + assertTrue(versionManager.isAllowed(HDDSVersion.SOFTWARE_VERSION), + "Software version should always be allowed when finalized"); + } + MetricsRecordBuilder metrics = getMetrics(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + assertGauge("SoftwareVersion", HDDSVersion.SOFTWARE_VERSION.serialize(), metrics); + assertGauge("ApparentVersion", apparentVersion.serialize(), metrics); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index cd0bb242460..9f69215b694 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -78,15 +78,6 @@ public int layoutVersion() { return layoutVersion; } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return serializedVersion >= layoutVersion() || - // Future versions support all known features. - OzoneManagerVersion.deserialize(serializedVersion).equals(OzoneManagerVersion.FUTURE_VERSION); - } - /** * @param version The serialized version to convert. * @return The version corresponding to this serialized value, or {@code null} if no matching version is From 9050949bf3c8b957ad3e849ce0ed978aade6d081 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:11:43 -0400 Subject: [PATCH 12/20] Make tests abstract and add passing implementation for OMVersionManager --- .../hdds/upgrade/TestHDDSVersionManager.java | 155 +++--------------- .../AbstractComponentVersionManagerTest.java | 147 +++++++++++++++++ .../om/upgrade/TestOMVersionManager.java | 68 ++++++++ 3 files changed, 242 insertions(+), 128 deletions(-) create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java index e6ac237300b..1e76ff91be3 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java @@ -1,51 +1,37 @@ /* - * licensed to the apache software foundation (asf) under one or more - * contributor license agreements. see the notice file distributed with + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * the asf licenses this file to you under the apache license, version 2.0 - * (the "license"); you may not use this file except in compliance with - * the license. you may obtain a copy of the license at + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/license-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * unless required by applicable law or agreed to in writing, software - * distributed under the license is distributed on an "as is" basis, - * without warranties or conditions of any kind, either express or implied. - * see the license for the specific language governing permissions and - * limitations under the license. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hdds.upgrade; -import static org.apache.ozone.test.MetricsAsserts.assertGauge; -import static org.apache.ozone.test.MetricsAsserts.getMetrics; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; import java.util.stream.Stream; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.ozone.upgrade.ComponentVersionManagerMetrics; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; +import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; -class TestHDDSVersionManager { +/** + * Tests for {@link HDDSVersionManager}. + */ +class TestHDDSVersionManager extends AbstractComponentVersionManagerTest { private static final List ALL_VERSIONS; @@ -60,111 +46,24 @@ class TestHDDSVersionManager { } } - /** - * @return All version less than our current software version that we can use as initial apparent versions to test - * finalizing to the latest software version. - */ - private static Stream preFinalizedVersionArgs() { + public static Stream preFinalizedVersionArgs() { return ALL_VERSIONS.stream() .limit(ALL_VERSIONS.size() - 1) .map(Arguments::of); } - @AfterEach - public void cleanupMetricsSource() { - DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); - } - - @Test - public void testApparentVersionTranslation() throws Exception { - for (ComponentVersion apparentVersion : ALL_VERSIONS) { - HDDSVersionManager versionManager = new HDDSVersionManager(apparentVersion.serialize()); - assertApparentVersion(versionManager, apparentVersion); - // Unregister the metrics for this object. A new one will be created on the next loop iteration. - DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); - } + @Override + protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { + return new HDDSVersionManager(serializedApparentVersion); } - @Test - public void testApparentVersionBehindSoftwareVersion() { - int serializedNextVersion = HDDSVersion.SOFTWARE_VERSION.serialize() + 1; - assertThrows(IOException.class, () -> new HDDSVersionManager(serializedNextVersion)); + @Override + protected List allVersionsInOrder() { + return ALL_VERSIONS; } - @ParameterizedTest - @MethodSource("preFinalizedVersionArgs") - public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) throws Exception { - int apparentVersionIndex = ALL_VERSIONS.indexOf(apparentVersion); - assertTrue(apparentVersionIndex >= 0, "apparentVersion must exist in preFinalizedVersions"); - Iterator expectedVersions = ALL_VERSIONS - .subList(apparentVersionIndex + 1, ALL_VERSIONS.size()).iterator(); - - HDDSVersionManager versionManager = new HDDSVersionManager(apparentVersion.serialize()); - assertApparentVersion(versionManager, apparentVersion); - - for (ComponentVersion versionToFinalize : versionManager.getUnfinalizedVersions()) { - assertTrue(versionManager.needsFinalization()); - assertFalse(versionManager.isAllowed(versionToFinalize), "Unfinalized version " + versionToFinalize + - " should not be allowed by apparent version " + versionManager.getApparentVersion()); - // Ensure versions are iterated in the expected order - assertTrue(expectedVersions.hasNext()); - assertEquals(expectedVersions.next(), versionToFinalize); - - versionManager.markFinalized(versionToFinalize); - - // This version should now be finalized. - assertApparentVersion(versionManager, versionToFinalize); - } - - assertFalse(expectedVersions.hasNext()); - assertThrows(NoSuchElementException.class, expectedVersions::next); - } - - @Test - public void testFinalizationFromSoftwareVersionNoOp() throws Exception { - HDDSVersionManager versionManager = new HDDSVersionManager(HDDSVersion.SOFTWARE_VERSION.serialize()); - - assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); - assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - - // Duplicate finalize call should not throw or change state. - versionManager.markFinalized(HDDSVersion.SOFTWARE_VERSION); - - assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); - assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - } - - @Test - public void testFinalizationOfNonExistentVersion() throws Exception { - HDDSVersionManager versionManager = new HDDSVersionManager(HDDSVersion.SOFTWARE_VERSION.serialize()); - assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); - assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - - // Create a mock version which appears newer than all existing versions, including the software version. - ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class); - when(mockVersion.isSupportedBy(any())).thenReturn(false); - - // Attempting to finalize this version should throw without changing state. - assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); - assertApparentVersion(versionManager, HDDSVersion.SOFTWARE_VERSION); - assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - } - - private static void assertApparentVersion(HDDSVersionManager versionManager, ComponentVersion apparentVersion) { - assertEquals(apparentVersion, versionManager.getApparentVersion()); - assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + " should be allowed"); - assertEquals(HDDSVersion.SOFTWARE_VERSION, versionManager.getSoftwareVersion(), - "Software version should never change"); - if (!versionManager.needsFinalization()) { - assertTrue(versionManager.isAllowed(HDDSVersion.SOFTWARE_VERSION), - "Software version should always be allowed when finalized"); - } - MetricsRecordBuilder metrics = getMetrics(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); - assertGauge("SoftwareVersion", HDDSVersion.SOFTWARE_VERSION.serialize(), metrics); - assertGauge("ApparentVersion", apparentVersion.serialize(), metrics); + @Override + protected ComponentVersion expectedSoftwareVersion() { + return HDDSVersion.SOFTWARE_VERSION; } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java new file mode 100644 index 00000000000..af3a308a92e --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import static org.apache.ozone.test.MetricsAsserts.assertGauge; +import static org.apache.ozone.test.MetricsAsserts.getMetrics; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; + +/** + * Shared tests for concrete {@link ComponentVersionManager} implementations. + */ +public abstract class AbstractComponentVersionManagerTest { + + protected abstract ComponentVersionManager createManager(int serializedApparentVersion) throws IOException; + + protected abstract List allVersionsInOrder(); + + protected abstract ComponentVersion expectedSoftwareVersion(); + + @AfterEach + public void cleanupMetricsSource() { + DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + } + + @Test + public void testApparentVersionTranslation() throws Exception { + for (ComponentVersion apparentVersion : allVersionsInOrder()) { + try (ComponentVersionManager versionManager = createManager(apparentVersion.serialize())) { + assertApparentVersion(versionManager, apparentVersion); + } + } + } + + @Test + public void testApparentVersionBehindSoftwareVersion() { + int serializedNextVersion = expectedSoftwareVersion().serialize() + 1; + assertThrows(IOException.class, () -> createManager(serializedNextVersion)); + } + + @ParameterizedTest + // Child classes must implement this as a static method to provide the versions to start finalization from. + @MethodSource("preFinalizedVersionArgs") + public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) throws Exception { + List allVersions = allVersionsInOrder(); + int apparentVersionIndex = allVersions.indexOf(apparentVersion); + assertTrue(apparentVersionIndex >= 0, "Apparent version " + apparentVersion + " must exist"); + Iterator expectedVersions = allVersions.subList(apparentVersionIndex + 1, allVersions.size()) + .iterator(); + + try (ComponentVersionManager versionManager = createManager(apparentVersion.serialize())) { + assertApparentVersion(versionManager, apparentVersion); + + for (ComponentVersion versionToFinalize : versionManager.getUnfinalizedVersions()) { + assertTrue(versionManager.needsFinalization()); + assertFalse(versionManager.isAllowed(versionToFinalize), + "Unfinalized version " + versionToFinalize + " should not be allowed by apparent version " + + versionManager.getApparentVersion()); + assertTrue(expectedVersions.hasNext()); + assertEquals(expectedVersions.next(), versionToFinalize); + + versionManager.markFinalized(versionToFinalize); + assertApparentVersion(versionManager, versionToFinalize); + } + + assertFalse(expectedVersions.hasNext()); + assertThrows(NoSuchElementException.class, expectedVersions::next); + } + } + + @Test + public void testFinalizationFromSoftwareVersionNoOp() throws Exception { + try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + versionManager.markFinalized(expectedSoftwareVersion()); + + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + } + + @Test + public void testFinalizationOfNonExistentVersion() throws Exception { + try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class); + when(mockVersion.isSupportedBy(any())).thenReturn(false); + + assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + } + + private void assertApparentVersion(ComponentVersionManager versionManager, ComponentVersion apparentVersion) { + assertEquals(apparentVersion, versionManager.getApparentVersion()); + assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + " should be allowed"); + assertEquals(expectedSoftwareVersion(), versionManager.getSoftwareVersion(), "Software version should never change"); + if (!versionManager.needsFinalization()) { + assertTrue(versionManager.isAllowed(expectedSoftwareVersion()), + "Software version should always be allowed when finalized"); + } + MetricsRecordBuilder metrics = getMetrics(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + assertGauge("SoftwareVersion", expectedSoftwareVersion().serialize(), metrics); + assertGauge("ApparentVersion", apparentVersion.serialize(), metrics); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java new file mode 100644 index 00000000000..daede09d1ca --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.junit.jupiter.params.provider.Arguments; + +/** + * Tests for {@link OMVersionManager}. + */ +class TestOMVersionManager extends AbstractComponentVersionManagerTest { + + private static final List ALL_VERSIONS; + + static { + ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values())); + for (OzoneManagerVersion version : OzoneManagerVersion.values()) { + // Add all defined versions after and including ZDU to get the complete version list. + if (OzoneManagerVersion.ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { + ALL_VERSIONS.add(version); + } + } + } + + private static Stream preFinalizedVersionArgs() { + return ALL_VERSIONS.stream() + .limit(ALL_VERSIONS.size() - 1) + .map(Arguments::of); + } + + @Override + protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { + return new OMVersionManager(serializedApparentVersion); + } + + @Override + protected List allVersionsInOrder() { + return ALL_VERSIONS; + } + + @Override + protected ComponentVersion expectedSoftwareVersion() { + return OzoneManagerVersion.SOFTWARE_VERSION; + } +} From d4a309da25c403f2f6663e9c8d56ea6c2b7dacaf Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:15:35 -0400 Subject: [PATCH 13/20] Remove not required isSupportedBy overrides --- .../main/java/org/apache/hadoop/ozone/ClientVersion.java | 7 ------- .../java/org/apache/hadoop/ozone/OzoneManagerVersion.java | 7 ------- .../ozone/upgrade/TestAbstractLayoutVersionManager.java | 6 ------ .../hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java | 6 ------ 4 files changed, 26 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index cf1059334bb..dd451ab052c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -82,13 +82,6 @@ public static ClientVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 2cca20ee9a7..163696b4f15 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -113,13 +113,6 @@ public OzoneManagerVersion nextVersion() { return values()[nextOrdinal]; } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java index 9761eaf8c73..6cd9391e28f 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -195,12 +195,6 @@ public ComponentVersion nextVersion() { // TODO HDDS-14826 will remove this test. No need to add handling for this new method. return null; } - - @Override - public boolean isSupportedBy(int serializedVersion) { - // TODO HDDS-14826 will remove this test. No need to add handling for this new method. - return false; - } }; } return lfs; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java index bee8134f799..dc3a3e6b268 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java @@ -92,12 +92,6 @@ public MockLayoutFeature nextVersion() { return nextOrdinal < values().length ? values()[nextOrdinal] : null; } - @Override - public boolean isSupportedBy(int serializedVersion) { - // TODO HDDS-14826 will remove this test. No need to add handling for this new method. - return false; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; From ba48a9df1858e8210ad4932f1c8f8ce3b17f7c8e Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:20:02 -0400 Subject: [PATCH 14/20] Clarify javadoc on serialize --- .../main/java/org/apache/hadoop/hdds/ComponentVersion.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 4bc511e37fc..a8950924c9a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -29,10 +29,10 @@ public interface ComponentVersion { /** * Returns an integer representation of this version. To callers outside this class, this is an opaque value which - * should not be checked or compared directly. + * should not be checked or compared directly. {@link #isSupportedBy} should be used for version comparisons. * - * To implementors of this interface, versions should serialize such that if version1 < version2, - * then version1.serialize() < version2.serialize(). + * To implementors of this interface, versions should serialize such that version1 <= version2 + * if and only if version1.serialize() <= version2.serialize(). * Negative numbers may be used as serialized values to represent unknown future versions which are trivially larger * than all other versions. * From f5711e38a4c219b6edc9edaf48591e31fbe34d09 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:28:43 -0400 Subject: [PATCH 15/20] Checkstyle --- .../apache/hadoop/hdds/ComponentVersion.java | 2 +- .../hdds/upgrade/HDDSLayoutFeature.java | 22 +++++++++---------- .../AbstractComponentVersionManagerTest.java | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index a8950924c9a..0f70321b825 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -41,7 +41,7 @@ public interface ComponentVersion { int serialize(); /** - * @return The description of the version enum value. + * @return The description of this version. */ String description(); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index dbca2bfe9ac..4e753c5cab3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -1,18 +1,18 @@ /* - * licensed to the apache software foundation (asf) under one or more - * contributor license agreements. see the notice file distributed with + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. - * the asf licenses this file to you under the apache license, version 2.0 - * (the "license"); you may not use this file except in compliance with - * the license. you may obtain a copy of the license at + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/license-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * unless required by applicable law or agreed to in writing, software - * distributed under the license is distributed on an "as is" basis, - * without warranties or conditions of any kind, either express or implied. - * see the license for the specific language governing permissions and - * limitations under the license. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hdds.upgrade; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java index af3a308a92e..09f26a10669 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java @@ -135,7 +135,8 @@ public void testFinalizationOfNonExistentVersion() throws Exception { private void assertApparentVersion(ComponentVersionManager versionManager, ComponentVersion apparentVersion) { assertEquals(apparentVersion, versionManager.getApparentVersion()); assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + " should be allowed"); - assertEquals(expectedSoftwareVersion(), versionManager.getSoftwareVersion(), "Software version should never change"); + assertEquals(expectedSoftwareVersion(), versionManager.getSoftwareVersion(), + "Software version should never change"); if (!versionManager.needsFinalization()) { assertTrue(versionManager.isAllowed(expectedSoftwareVersion()), "Software version should always be allowed when finalized"); From 0d05b73497a4e5c818ac7f93b9df4eace0f618b2 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:31:20 -0400 Subject: [PATCH 16/20] Undo unnecessary debug change --- .../org/apache/hadoop/ozone/debug/VersionDebug.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java index f9daf5afcb5..2112a7fac06 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.concurrent.Callable; import org.apache.hadoop.hdds.ComponentVersion; -import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.hdds.DatanodeVersion; import org.apache.hadoop.hdds.cli.DebugSubcommand; import org.apache.hadoop.hdds.server.JsonUtils; import org.apache.hadoop.ozone.ClientVersion; @@ -52,19 +52,18 @@ public Void call() throws IOException { ), "components", ImmutableSortedMap.of( "client", asMap(ClientVersion.CURRENT), - "datanode", asMap(HDDSVersion.SOFTWARE_VERSION), - "om", asMap(OzoneManagerVersion.SOFTWARE_VERSION) + "datanode", asMap(DatanodeVersion.CURRENT), + "om", asMap(OzoneManagerVersion.CURRENT) ) ))); return null; } - private static & ComponentVersion> - Map asMap(T version) { + private static & ComponentVersion> Map asMap(T version) { return ImmutableSortedMap.of( "componentVersion", ImmutableSortedMap.of( "name", version.name(), - "protoValue", version.serialize() + "protoValue", version.toProtoValue() ) ); } From 5deb4b9158dbb8da0fd0c146b0a9d252a2ab3972 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 16:45:37 -0400 Subject: [PATCH 17/20] Undo no longer necessary validator changes --- .../ozone/annotations/RegisterValidatorProcessor.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index 3e5456f9b3a..c5639e11e54 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -29,7 +29,6 @@ import javax.lang.model.element.TypeElement; import javax.lang.model.type.ArrayType; import javax.lang.model.type.TypeKind; -import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import javax.lang.model.util.Types; import javax.tools.Diagnostic; @@ -55,7 +54,7 @@ public class RegisterValidatorProcessor extends AbstractProcessor { public static final String ANNOTATION_SIMPLE_NAME = "RegisterValidator"; - public static final String VERSION_CLASS_NAME = "org.apache.hadoop.hdds.ComponentVersion"; + public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version"; public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" + ".RequestProcessingPhase"; public static final String APPLY_BEFORE_METHOD_NAME = "applyBefore"; @@ -87,13 +86,11 @@ private boolean validateArrayMethod(ExecutableElement method, String expectedMet Types types = processingEnv.getTypeUtils(); TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : elementUtils.getTypeElement(expectedReturnClass); - TypeMirror expectedType = expectedReturnInterface == null - ? null : types.erasure(expectedReturnInterface.asType()); return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || TypeKind.ARRAY.equals(method.getReturnType().getKind()) && types.asElement(((ArrayType)method.getReturnType()).getComponentType()).getKind() == expectedReturnType) && (expectedReturnInterface == null || - types.isAssignable(types.erasure(types.asElement(method.getReturnType()).asType()), expectedType)); + types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); } private boolean validateMethod(ExecutableElement method, String expectedMethodName, ElementKind expectedReturnType, @@ -102,13 +99,11 @@ private boolean validateMethod(ExecutableElement method, String expectedMethodNa Types types = processingEnv.getTypeUtils(); TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : elementUtils.getTypeElement(expectedReturnClass); - TypeMirror expectedType = expectedReturnInterface == null - ? null : types.erasure(expectedReturnInterface.asType()); return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || types.asElement(method.getReturnType()) != null && types.asElement(method.getReturnType()).getKind() == expectedReturnType) && (expectedReturnInterface == null || - types.isAssignable(types.erasure(types.asElement(method.getReturnType()).asType()), expectedType)); + types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); } private void processElements(Set annotatedElements) { From 626a53657fa1f04330af0dab20df7b8822b56b50 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 17:46:23 -0400 Subject: [PATCH 18/20] Updates after reviewing the diff --- .../hdds/upgrade/TestHDDSLayoutVersionManager.java | 10 ++++++++++ .../upgrade/AbstractComponentVersionManagerTest.java | 1 + .../ozone/upgrade/TestUpgradeFinalizerActions.java | 5 +++-- .../org/apache/hadoop/ozone/debug/VersionDebug.java | 7 ++++--- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java index 37d3b88eeb1..4792e1179de 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutVersionManager.java @@ -82,4 +82,14 @@ public void testUpgradeActionsRegistered() throws Exception { verify(mockObj, times(0)).mockMethodScm(); verify(mockObj, times(1)).mockMethodDn(); } + + @Test + public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() { + HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); + int currVersion = -1; + for (HDDSLayoutFeature lf : values) { + assertEquals(currVersion + 1, lf.layoutVersion()); + currVersion = lf.layoutVersion(); + } + } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java index 09f26a10669..0bf439326ca 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java @@ -126,6 +126,7 @@ public void testFinalizationOfNonExistentVersion() throws Exception { when(mockVersion.isSupportedBy(any())).thenReturn(false); assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); + // The failed finalization call should not have changed the version manager's state. assertApparentVersion(versionManager, expectedSoftwareVersion()); assertFalse(versionManager.needsFinalization()); assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java index dc3a3e6b268..89a52b7c31d 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java @@ -88,8 +88,9 @@ public String description() { @Override public MockLayoutFeature nextVersion() { - int nextOrdinal = ordinal() + 1; - return nextOrdinal < values().length ? values()[nextOrdinal] : null; + // TODO HDDS-14826 will remove the tests that are using this. No need to provide an implementation for this new + // method. + return null; } @Override diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java index 2112a7fac06..771316d4553 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java @@ -23,6 +23,7 @@ import java.util.concurrent.Callable; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.DatanodeVersion; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.cli.DebugSubcommand; import org.apache.hadoop.hdds.server.JsonUtils; import org.apache.hadoop.ozone.ClientVersion; @@ -52,8 +53,8 @@ public Void call() throws IOException { ), "components", ImmutableSortedMap.of( "client", asMap(ClientVersion.CURRENT), - "datanode", asMap(DatanodeVersion.CURRENT), - "om", asMap(OzoneManagerVersion.CURRENT) + "datanode", asMap(HDDSVersion.SOFTWARE_VERSION), + "om", asMap(OzoneManagerVersion.SOFTWARE_VERSION) ) ))); return null; @@ -63,7 +64,7 @@ private static & ComponentVersion> Map asMap( return ImmutableSortedMap.of( "componentVersion", ImmutableSortedMap.of( "name", version.name(), - "protoValue", version.toProtoValue() + "protoValue", version.serialize() ) ); } From 729ba2b5d463adc38ac4eb06d3f0c66178aacc27 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 18:01:49 -0400 Subject: [PATCH 19/20] Fix build --- .../main/java/org/apache/hadoop/ozone/debug/VersionDebug.java | 1 - .../apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java index 771316d4553..87cf936deb3 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/VersionDebug.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.concurrent.Callable; import org.apache.hadoop.hdds.ComponentVersion; -import org.apache.hadoop.hdds.DatanodeVersion; import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.cli.DebugSubcommand; import org.apache.hadoop.hdds.server.JsonUtils; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java index daede09d1ca..eed52733acb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -45,7 +45,7 @@ class TestOMVersionManager extends AbstractComponentVersionManagerTest { } } - private static Stream preFinalizedVersionArgs() { + public static Stream preFinalizedVersionArgs() { return ALL_VERSIONS.stream() .limit(ALL_VERSIONS.size() - 1) .map(Arguments::of); From 9a763d75ebfd6698263b212778dec6a13ef557f3 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 13 Mar 2026 18:19:00 -0400 Subject: [PATCH 20/20] Fix javadoc warnings --- .../main/java/org/apache/hadoop/hdds/ComponentVersion.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 0f70321b825..9c1223f35c3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -31,8 +31,9 @@ public interface ComponentVersion { * Returns an integer representation of this version. To callers outside this class, this is an opaque value which * should not be checked or compared directly. {@link #isSupportedBy} should be used for version comparisons. * - * To implementors of this interface, versions should serialize such that version1 <= version2 - * if and only if version1.serialize() <= version2.serialize(). + * To implementors of this interface, versions should serialize such that + * {@code version1 <= version2} if and only if + * {@code version1.serialize() <= version2.serialize()}. * Negative numbers may be used as serialized values to represent unknown future versions which are trivially larger * than all other versions. *