diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index 6b26043efb26..c5639e11e540 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -54,7 +54,7 @@ public class RegisterValidatorProcessor extends AbstractProcessor { public static final String ANNOTATION_SIMPLE_NAME = "RegisterValidator"; - public static final String VERSION_CLASS_NAME = "org.apache.hadoop.hdds.ComponentVersion"; + public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version"; public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" + ".RequestProcessingPhase"; public static final String APPLY_BEFORE_METHOD_NAME = "applyBefore"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 10f232ee0c90..9c1223f35c30 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -21,26 +21,61 @@ import org.apache.hadoop.ozone.upgrade.UpgradeAction; /** - * Base type for component version enums. + * The logical versioning system used to track incompatible changes to a component, regardless whether they affect disk + * or network compatibility between the same or different types of components. + * + * This interface is the base type for component version enums. */ public interface ComponentVersion { /** - * @return The serialized representation of this version. This is an opaque value which should not be checked or - * compared directly. + * Returns an integer representation of this version. To callers outside this class, this is an opaque value which + * should not be checked or compared directly. {@link #isSupportedBy} should be used for version comparisons. + * + * To implementors of this interface, versions should serialize such that + * {@code version1 <= version2} if and only if + * {@code version1.serialize() <= version2.serialize()}. + * Negative numbers may be used as serialized values to represent unknown future versions which are trivially larger + * than all other versions. + * + * @return The serialized representation of this version. */ int serialize(); /** - * @return the description of the version enum value. + * @return The description of this version. */ String description(); /** - * Deserializes a ComponentVersion and checks if its feature set is supported by the current ComponentVersion. + * @return The next version immediately following this one, or null if there is no such version. + */ + ComponentVersion nextVersion(); + + /** + * Uses the serialized representation of a ComponentVersion to check if its feature set is supported by the current + * ComponentVersion. * - * @return true if this version supports the features of otherVersion. False otherwise. + * @return true if this version supports the features of the provided version. False otherwise. */ - boolean isSupportedBy(int serializedVersion); + default boolean isSupportedBy(int serializedVersion) { + if (serialize() < 0) { + // Our version is an unknown future version, it is not supported by any other version. + return false; + } else if (serializedVersion < 0) { + // The other version is an unknown future version, it trivially supports all other versions. + return true; + } else { + // If both versions have positive values, they represent concrete versions and we can compare them directly. + return serialize() <= serializedVersion; + } + } + + /** + * @return true if this version supports the features of the provided version. False otherwise. + */ + default boolean isSupportedBy(ComponentVersion other) { + return isSupportedBy(other.serialize()); + } default Optional action() { return Optional.empty(); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java index 6b4131e22265..51d4229a7484 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HDDSVersion.java @@ -21,13 +21,16 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; /** * Versioning for datanode. */ public enum HDDSVersion implements ComponentVersion { + ////////////////////////////// ////////////////////////////// + DEFAULT_VERSION(0, "Initial version"), SEPARATE_RATIS_PORTS_AVAILABLE(1, "Version with separated Ratis port."), @@ -36,14 +39,18 @@ public enum HDDSVersion implements ComponentVersion { STREAM_BLOCK_SUPPORT(3, "This version has support for reading a block by streaming chunks."), + ZDU(100, "Version that supports zero downtime upgrade"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); - public static final HDDSVersion SOFTWARE_VERSION = latest(); + ////////////////////////////// ////////////////////////////// - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(HDDSVersion::serialize, identity())); + .collect(toMap(HDDSVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final HDDSVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -58,31 +65,35 @@ public String description() { return description; } + /** + * @return The next version immediately following this one and excluding FUTURE_VERSION, + * or null if there is no such version. + */ + @Override + public HDDSVersion nextVersion() { + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; + } + @Override public int serialize() { return version; } + /** + * @param value The serialized version to convert. + * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * found. + */ public static HDDSVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; } - - private static HDDSVersion latest() { - HDDSVersion[] versions = HDDSVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java index e78e1dcbffe9..4e753c5cab32 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java @@ -17,11 +17,21 @@ package org.apache.hadoop.hdds.upgrade; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.ozone.upgrade.LayoutFeature; /** - * List of HDDS Features. + * List of HDDS Layout Features. All version management has been migrated to {@link HDDSVersion} and no new additions + * should be made to this class. Existing versions are kept here for backwards compatibility when upgrading to this + * version from older versions. */ public enum HDDSLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// @@ -44,8 +54,14 @@ public enum HDDSLayoutFeature implements LayoutFeature { WITNESSED_CONTAINER_DB_PROTO_VALUE(9, "ContainerID table schema to use value type as proto"), STORAGE_SPACE_DISTRIBUTION(10, "Enhanced block deletion function for storage space distribution feature."); + // ALL NEW VERSIONS SHOULD NOW BE ADDED TO HDDSVersion + ////////////////////////////// ////////////////////////////// + private static final SortedMap BY_VALUE = + Arrays.stream(values()) + .collect(toMap(HDDSLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + private final int layoutVersion; private final String description; private HDDSUpgradeAction scmAction; @@ -90,6 +106,30 @@ public String description() { return description; } + /** + * @return The next version immediately following this one. If there is no next version found in this enum, + * the next version is {@link HDDSVersion#ZDU}, since all HDDS versioning has been migrated to + * {@link HDDSVersion} as part of the ZDU feature. + */ + @Override + public ComponentVersion nextVersion() { + HDDSLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1); + if (nextFeature == null) { + return HDDSVersion.ZDU; + } else { + return nextFeature; + } + } + + /** + * @param version The serialized version to convert. + * @return The version corresponding to this serialized value, or {@code null} if no matching version is + * found. + */ + public static HDDSLayoutFeature deserialize(int version) { + return BY_VALUE.get(version); + } + @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index ac20b5fe3b77..dd451ab052cb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -21,7 +21,8 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; /** @@ -44,11 +45,11 @@ public enum ClientVersion implements ComponentVersion { FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); - public static final ClientVersion CURRENT = latest(); - - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(ClientVersion::serialize, identity())); + .collect(toMap(ClientVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final ClientVersion CURRENT = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -63,6 +64,15 @@ public String description() { return description; } + @Override + public ClientVersion nextVersion() { + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; + } + @Override public int serialize() { return version; @@ -72,22 +82,8 @@ public static ClientVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } - @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; - } - @Override public String toString() { return name() + " (" + serialize() + ")"; } - - private static ClientVersion latest() { - ClientVersion[] versions = ClientVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index 9ec8870855ec..163696b4f15f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -21,13 +21,17 @@ import static java.util.stream.Collectors.toMap; import java.util.Arrays; -import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import org.apache.hadoop.hdds.ComponentVersion; /** * Versioning for Ozone Manager. */ public enum OzoneManagerVersion implements ComponentVersion { + + ////////////////////////////// ////////////////////////////// + DEFAULT_VERSION(0, "Initial version"), S3G_PERSISTENT_CONNECTIONS(1, "New S3G persistent connection support is present in OM."), @@ -54,15 +58,19 @@ public enum OzoneManagerVersion implements ComponentVersion { S3_LIST_MULTIPART_UPLOADS_PAGINATION(11, "OzoneManager version that supports S3 list multipart uploads API with pagination"), - + + ZDU(100, "OzoneManager version that supports zero downtime upgrade"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); - public static final OzoneManagerVersion SOFTWARE_VERSION = latest(); + ////////////////////////////// ////////////////////////////// - private static final Map BY_VALUE = + private static final SortedMap BY_VALUE = Arrays.stream(values()) - .collect(toMap(OzoneManagerVersion::serialize, identity())); + .collect(toMap(OzoneManagerVersion::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + + public static final OzoneManagerVersion SOFTWARE_VERSION = BY_VALUE.get(BY_VALUE.lastKey()); private final int version; private final String description; @@ -82,26 +90,31 @@ public int serialize() { return version; } + /** + * @param value The serialized version to convert. + * @return The version corresponding to this serialized value, or {@link #FUTURE_VERSION} if no matching version is + * found. + */ public static OzoneManagerVersion deserialize(int value) { return BY_VALUE.getOrDefault(value, FUTURE_VERSION); } + + /** + * @return The next version immediately following this one and excluding FUTURE_VERSION, + * or null if there is no such version. + */ @Override - public boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - return deserialize(serializedVersion).compareTo(this) >= 0; + public OzoneManagerVersion nextVersion() { + int nextOrdinal = ordinal() + 1; + if (nextOrdinal >= values().length - 1) { + return null; + } + return values()[nextOrdinal]; } @Override public String toString() { return name() + " (" + serialize() + ")"; } - - private static OzoneManagerVersion latest() { - OzoneManagerVersion[] versions = OzoneManagerVersion.values(); - // The last entry in the array will be `FUTURE_VERSION`. We want the entry prior to this which defines the latest - // version in the software. - return versions[versions.length - 2]; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 848a35104e5f..7f7374c8137a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -29,12 +29,4 @@ public interface LayoutFeature extends ComponentVersion { default int serialize() { return this.layoutVersion(); } - - @Override - default boolean isSupportedBy(int serializedVersion) { - // In order for the other serialized version to support this version's features, - // the other version must be equal or larger to this version. - // We can compare the values directly since there is no FUTURE_VERSION for layout features. - return serializedVersion >= layoutVersion(); - } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java new file mode 100644 index 000000000000..bfa534629a54 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/AbstractComponentVersionTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * Shared invariants for component version enums. + */ +public abstract class AbstractComponentVersionTest { + + protected abstract ComponentVersion[] getValues(); + + protected abstract ComponentVersion getDefaultVersion(); + + protected abstract ComponentVersion getFutureVersion(); + + protected abstract ComponentVersion deserialize(int value); + + // FUTURE_VERSION is the latest + @Test + public void testFutureVersionHasTheHighestOrdinal() { + ComponentVersion[] values = getValues(); + ComponentVersion futureValue = getFutureVersion(); + assertEquals(values[values.length - 1], futureValue); + } + + // FUTURE_VERSION's internal version id is -1 + @Test + public void testFutureVersionSerializesToMinusOne() { + ComponentVersion futureValue = getFutureVersion(); + assertEquals(-1, futureValue.serialize()); + } + + // DEFAULT_VERSION's internal version id is 0 + @Test + public void testDefaultVersionSerializesToZero() { + ComponentVersion defaultValue = getDefaultVersion(); + assertEquals(0, defaultValue.serialize()); + } + + // known (non-future) versions are strictly increasing + @Test + public void testSerializedValuesAreMonotonic() { + ComponentVersion[] values = getValues(); + int knownVersionCount = values.length - 1; + for (int i = 1; i < knownVersionCount; i++) { + assertTrue(values[i].serialize() > values[i - 1].serialize(), + "Expected known version serialization to increase: " + values[i - 1] + " -> " + values[i]); + } + } + + @Test + public void testNextVersionProgression() { + ComponentVersion[] values = getValues(); + ComponentVersion futureValue = getFutureVersion(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + assertEquals(values[i + 1], values[i].nextVersion(), + "Expected nextVersion progression for " + values[i]); + } + assertNull(values[knownVersionCount - 1].nextVersion(), + "Expected latest known version to have no nextVersion"); + assertNull(futureValue.nextVersion(), + "Expected FUTURE_VERSION.nextVersion() to return null"); + } + + @Test + public void testOnlyEqualOrHigherVersionsCanSupportAFeature() { + ComponentVersion[] values = getValues(); + int knownVersionCount = values.length - 1; + for (int featureIndex = 0; featureIndex < knownVersionCount; featureIndex++) { + ComponentVersion requiredFeature = values[featureIndex]; + for (int providerIndex = 0; providerIndex < knownVersionCount; providerIndex++) { + ComponentVersion provider = values[providerIndex]; + if (providerIndex >= featureIndex) { + assertSupportedBy(requiredFeature, provider); + } else { + assertNotSupportedBy(requiredFeature, provider); + } + } + } + } + + @Test + public void testFutureVersionSupportsAllKnownVersions() { + ComponentVersion[] values = getValues(); + int unknownFutureVersion = Integer.MAX_VALUE; + for (ComponentVersion requiredFeature : values) { + assertTrue(requiredFeature.isSupportedBy(unknownFutureVersion)); + } + } + + @Test + public void testVersionSerDes() { + for (ComponentVersion version : getValues()) { + assertEquals(version, deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownReturnsFutureVersion() { + assertEquals(getFutureVersion(), deserialize(Integer.MAX_VALUE)); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java new file mode 100644 index 000000000000..0531523cfac4 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/ComponentVersionTestUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Shared assertions for {@link ComponentVersion} tests. + */ +public final class ComponentVersionTestUtils { + + private ComponentVersionTestUtils() { } + + public static void assertSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider) { + assertSupportedBy(requiredFeature, provider, true); + } + + public static void assertNotSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider) { + assertSupportedBy(requiredFeature, provider, false); + } + + /** + * Helper method to test support by passing both serialized and deserialized versions. + */ + private static void assertSupportedBy( + ComponentVersion requiredFeature, ComponentVersion provider, boolean expected) { + int providerSerializedVersion = provider.serialize(); + assertEquals(expected, requiredFeature.isSupportedBy(providerSerializedVersion), + "Expected support check via serialized overload to match for version " + + providerSerializedVersion); + assertEquals(expected, requiredFeature.isSupportedBy(provider), + "Expected support check via version overload to match for " + provider); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java new file mode 100644 index 000000000000..721fb2466c9a --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestClientVersion.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import org.apache.hadoop.ozone.ClientVersion; + +/** + * Invariants for {@link ClientVersion}. + */ +public class TestClientVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return ClientVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return ClientVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return ClientVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return ClientVersion.deserialize(value); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java deleted file mode 100644 index 0edab7e76d2f..000000000000 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestComponentVersionInvariants.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; - -import java.util.stream.Stream; -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -/** - * Test to ensure Component version instances conform with invariants relied - * upon in other parts of the codebase. - */ -public class TestComponentVersionInvariants { - - public static Stream values() { - return Stream.of( - arguments( - HDDSVersion.values(), - HDDSVersion.DEFAULT_VERSION, - HDDSVersion.FUTURE_VERSION), - arguments( - ClientVersion.values(), - ClientVersion.DEFAULT_VERSION, - ClientVersion.FUTURE_VERSION), - arguments( - OzoneManagerVersion.values(), - OzoneManagerVersion.DEFAULT_VERSION, - OzoneManagerVersion.FUTURE_VERSION) - ); - } - - // FUTURE_VERSION is the latest - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionHasTheHighestOrdinal(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - - assertEquals(values[values.length - 1], futureValue); - } - - // FUTURE_VERSION's internal version id is -1 - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionSerializesToMinusOne(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - assertEquals(-1, futureValue.serialize()); - - } - - // DEFAULT_VERSION's internal version id is 0 - @ParameterizedTest - @MethodSource("values") - public void testDefaultVersionSerializesToZero(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - assertEquals(0, defaultValue.serialize()); - } - - // versions are increasing monotonically by one - @ParameterizedTest - @MethodSource("values") - public void testSerializedValuesAreMonotonic(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int startValue = defaultValue.serialize(); - // we skip the future version at the last position - for (int i = 0; i < values.length - 1; i++) { - assertEquals(values[i].serialize(), startValue++); - } - assertEquals(values.length, ++startValue); - } - - @ParameterizedTest - @MethodSource("values") - public void testVersionIsSupportedByItself(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - for (ComponentVersion value : values) { - assertTrue(value.isSupportedBy(value.serialize())); - } - } - - @ParameterizedTest - @MethodSource("values") - public void testOnlyEqualOrHigherVersionsCanSupportAFeature(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int knownVersionCount = values.length - 1; - for (int featureIndex = 0; featureIndex < knownVersionCount; featureIndex++) { - ComponentVersion requiredFeature = values[featureIndex]; - for (int providerIndex = 0; providerIndex < knownVersionCount; providerIndex++) { - ComponentVersion provider = values[providerIndex]; - boolean expected = providerIndex >= featureIndex; - assertEquals(expected, requiredFeature.isSupportedBy(provider.serialize())); - } - } - } - - @ParameterizedTest - @MethodSource("values") - public void testFutureVersionSupportsAllKnownVersions(ComponentVersion[] values, ComponentVersion defaultValue, - ComponentVersion futureValue) { - int unknownFutureVersion = Integer.MAX_VALUE; - for (ComponentVersion requiredFeature : values) { - assertTrue(requiredFeature.isSupportedBy(unknownFutureVersion)); - } - } - - @Test - public void testHDDSVersionSerDes() { - for (HDDSVersion version: HDDSVersion.values()) { - assertEquals(version, HDDSVersion.deserialize(version.serialize())); - } - } - - @Test - public void testOMVersionSerDes() { - for (OzoneManagerVersion version: OzoneManagerVersion.values()) { - assertEquals(version, OzoneManagerVersion.deserialize(version.serialize())); - } - } - - @Test - public void testClientVersionSerDes() { - for (ClientVersion version: ClientVersion.values()) { - assertEquals(version, ClientVersion.deserialize(version.serialize())); - } - } -} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java new file mode 100644 index 000000000000..89d24d3c82ea --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHDDSVersion.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Invariants for {@link HDDSVersion}. + */ +public class TestHDDSVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return HDDSVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return HDDSVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return HDDSVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return HDDSVersion.deserialize(value); + } + + @Test + public void testKnownVersionNumbersAreContiguousExceptForZDU() { + HDDSVersion[] values = HDDSVersion.values(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + HDDSVersion current = values[i]; + HDDSVersion next = values[i + 1]; + if (next == HDDSVersion.ZDU) { + assertEquals(100, next.serialize()); + } else { + assertEquals(current.serialize() + 1, next.serialize()); + } + } + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java new file mode 100644 index 000000000000..bb94ac938fde --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestOzoneManagerVersion.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.junit.jupiter.api.Test; + +/** + * Invariants for {@link OzoneManagerVersion}. + */ +public class TestOzoneManagerVersion extends AbstractComponentVersionTest { + + @Override + protected ComponentVersion[] getValues() { + return OzoneManagerVersion.values(); + } + + @Override + protected ComponentVersion getDefaultVersion() { + return OzoneManagerVersion.DEFAULT_VERSION; + } + + @Override + protected ComponentVersion getFutureVersion() { + return OzoneManagerVersion.FUTURE_VERSION; + } + + @Override + protected ComponentVersion deserialize(int value) { + return OzoneManagerVersion.deserialize(value); + } + + @Test + public void testKnownVersionNumbersAreContiguousExceptForZDU() { + OzoneManagerVersion[] values = OzoneManagerVersion.values(); + int knownVersionCount = values.length - 1; + for (int i = 0; i < knownVersionCount - 1; i++) { + OzoneManagerVersion current = values[i]; + OzoneManagerVersion next = values[i + 1]; + if (next == OzoneManagerVersion.ZDU) { + assertEquals(100, next.serialize()); + } else { + assertEquals(current.serialize() + 1, next.serialize()); + } + } + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java new file mode 100644 index 000000000000..fbb89220e976 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSLayoutFeature.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.upgrade; + +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hadoop.hdds.HDDSVersion; +import org.junit.jupiter.api.Test; + +/** + * Tests invariants for legacy HDDS layout feature versions. + */ +public class TestHDDSLayoutFeature { + @Test + public void testHDDSLayoutFeaturesHaveIncreasingLayoutVersion() { + HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); + int currVersion = -1; + for (HDDSLayoutFeature lf : values) { + // This will skip the jump from the last HDDSLayoutFeature to HDDSVersion#ZDU, + // since that is expected to be a larger version increment. + assertEquals(currVersion + 1, lf.layoutVersion(), + "Expected monotonically increasing layout version for " + lf); + currVersion = lf.layoutVersion(); + } + } + + /** + * All incompatible changes to HDDS (SCM and Datanodes) should now be added to {@link HDDSVersion}. + */ + @Test + public void testNoNewHDDSLayoutFeaturesAdded() { + int numHDDSLayoutFeatures = HDDSLayoutFeature.values().length; + HDDSLayoutFeature lastFeature = HDDSLayoutFeature.values()[numHDDSLayoutFeatures - 1]; + assertEquals(11, numHDDSLayoutFeatures); + assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION, lastFeature); + assertEquals(10, lastFeature.layoutVersion()); + } + + @Test + public void testNextVersion() { + HDDSLayoutFeature[] values = HDDSLayoutFeature.values(); + for (int i = 1; i < values.length; i++) { + HDDSLayoutFeature previous = values[i - 1]; + HDDSLayoutFeature current = values[i]; + assertEquals(current, previous.nextVersion(), + "Expected " + previous + ".nextVersion() to be " + current); + } + // The last layout feature should point us to the ZDU version to switch to using HDDSVersion. + assertEquals(HDDSVersion.ZDU, values[values.length - 1].nextVersion()); + } + + @Test + public void testSerDes() { + for (HDDSLayoutFeature version : HDDSLayoutFeature.values()) { + assertEquals(version, HDDSLayoutFeature.deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownVersionReturnsNull() { + assertNull(HDDSLayoutFeature.deserialize(-1)); + assertNull(HDDSLayoutFeature.deserialize(Integer.MAX_VALUE)); + // HDDSLayoutFeature can only deserialize values from its own enum. + assertNull(HDDSLayoutFeature.deserialize(HDDSVersion.ZDU.serialize())); + } + + @Test + public void testIsSupportedByFeatureBoundary() { + for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { + // A layout feature should support itself. + int layoutVersion = feature.layoutVersion(); + assertSupportedBy(feature, feature); + if (layoutVersion > 0) { + // A layout feature should not be supported by older features. + HDDSLayoutFeature previousFeature = HDDSLayoutFeature.values()[layoutVersion - 1]; + assertNotSupportedBy(feature, previousFeature); + } + } + } + + @Test + public void testAllLayoutFeaturesAreSupportedByFutureVersions() { + for (HDDSLayoutFeature feature : HDDSLayoutFeature.values()) { + assertSupportedBy(feature, HDDSVersion.ZDU); + assertSupportedBy(feature, HDDSVersion.FUTURE_VERSION); + // No ComponentVersion instance represents an arbitrary future version. + assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); + } + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java new file mode 100644 index 000000000000..ad6bad1c7029 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; + +/** + * Component version manager for HDDS (Datanodes and SCM). + */ +public class HDDSVersionManager extends ComponentVersionManager { + public HDDSVersionManager(int serializedApparentVersion) throws IOException { + super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION); + } + + /** + * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link HDDSVersion}. + * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link HDDSLayoutFeature}. + */ + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { + if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) { + return HDDSLayoutFeature.deserialize(serializedApparentVersion); + } else { + return HDDSVersion.deserialize(serializedApparentVersion); + } + } + + // TODO HDDS-14826: Register upgrade actions based on annotations +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java new file mode 100644 index 000000000000..11bf4a3f2281 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks information about the apparent version, software version, and finalization status of a component. + * + * Software version: The {@link ComponentVersion} of the code that is currently running. + * This is always the highest component version within the code and does not change while the process is running. + * + * Apparent version: The {@link ComponentVersion} the software is acting as, which is persisted to the disk. + * The apparent version determines the API that is exposed by the component and the format it uses to persist data. + * Using an apparent version less than software version allows us to support rolling upgrades and downgrades. + * + * Pre-finalized: State a component enters when the apparent version on disk is less than the software version. + * At this time all other machines may or may not be running the new bits, new features are blocked, and downgrade + * is allowed. + * + * Finalized: State a component enters when the apparent version is equal to the software version. + * A component transitions from pre-finalized to finalized when it receives a finalize command from the + * admin. At this time all machines are running the new bits, and even though this component is finalized, + * different types of components may not be. Downgrade is not allowed after this point. + * + */ +public abstract class ComponentVersionManager implements Closeable { + // Apparent version may be updated during the finalization process. + private volatile ComponentVersion apparentVersion; + // Software version will never change. + private final ComponentVersion softwareVersion; + private final ComponentVersionManagerMetrics metrics; + + private static final Logger LOG = + LoggerFactory.getLogger(ComponentVersionManager.class); + + protected ComponentVersionManager(ComponentVersion apparentVersion, ComponentVersion softwareVersion) + throws IOException { + this.apparentVersion = apparentVersion; + this.softwareVersion = softwareVersion; + + if (!apparentVersion.isSupportedBy(softwareVersion)) { + throw new IOException( + "Cannot initialize ComponentVersionManager. Apparent version " + + apparentVersion + " is larger than software version " + + softwareVersion); + } + + LOG.info("Initializing version manager with apparent version {} and software version {}", + apparentVersion, softwareVersion); + this.metrics = ComponentVersionManagerMetrics.create(this); + } + + public ComponentVersion getApparentVersion() { + return apparentVersion; + } + + public ComponentVersion getSoftwareVersion() { + return softwareVersion; + } + + public boolean isAllowed(ComponentVersion version) { + return version.isSupportedBy(apparentVersion); + } + + public boolean needsFinalization() { + return !apparentVersion.equals(softwareVersion); + } + + /** + * @return An Iterable of all versions after the current apparent version which still need to be finalized. If this + * component is already finalized, the Iterable will be empty. + */ + public Iterable getUnfinalizedVersions() { + return () -> new Iterator() { + private ComponentVersion currentVersion = apparentVersion; + + @Override + public boolean hasNext() { + return currentVersion.nextVersion() != null; + } + + @Override + public ComponentVersion next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + currentVersion = currentVersion.nextVersion(); + return currentVersion; + } + }; + } + + /** + * Validates that the provided version is valid to finalize to, and if so, updates the in-memory apparent version to + * this version. Also logs corresponding messages about finalization status. + * + * @param newApparentVersion The version to mark as finalized. + */ + public void markFinalized(ComponentVersion newApparentVersion) { + String versionMsg = "Software version: " + softwareVersion + + ", apparent version: " + apparentVersion + + ", provided version: " + newApparentVersion + + "."; + + if (newApparentVersion.isSupportedBy(apparentVersion)) { + LOG.info("Finalize attempt on a version which has already been finalized. {} This can happen when " + + "Raft Log is replayed during service restart.", versionMsg); + } else { + ComponentVersion nextVersion = apparentVersion.nextVersion(); + if (nextVersion == null) { + throw new IllegalArgumentException("Attempt to finalize when no future versions exist." + versionMsg); + } else if (nextVersion.equals(newApparentVersion)) { + apparentVersion = newApparentVersion; + LOG.info("Version {} has been finalized.", apparentVersion); + if (!needsFinalization()) { + LOG.info("Finalization is complete."); + } + } else { + throw new IllegalArgumentException( + "Finalize attempt on a version that is newer than the next feature to be finalized. " + versionMsg); + } + } + } + + @Override + public void close() { + metrics.unRegister(); + } +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java new file mode 100644 index 000000000000..3cafd99d48e9 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManagerMetrics.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.ozone.OzoneConsts; + +/** + * Metrics for {@link ComponentVersionManager}. + */ +@Metrics(about = "Component Version Manager Metrics", context = OzoneConsts.OZONE) +public final class ComponentVersionManagerMetrics implements MetricsSource { + + public static final String METRICS_SOURCE_NAME = + ComponentVersionManagerMetrics.class.getSimpleName(); + + private static final MetricsInfo SOFTWARE_VERSION = Interns.info( + "SoftwareVersion", + "Software version in serialized int form."); + private static final MetricsInfo APPARENT_VERSION = Interns.info( + "ApparentVersion", + "Current apparent version in serialized int form."); + + private final ComponentVersionManager versionManager; + + private ComponentVersionManagerMetrics(ComponentVersionManager versionManager) { + this.versionManager = versionManager; + } + + public static ComponentVersionManagerMetrics create(ComponentVersionManager versionManager) { + ComponentVersionManagerMetrics metrics = (ComponentVersionManagerMetrics) DefaultMetricsSystem.instance() + .getSource(METRICS_SOURCE_NAME); + if (metrics == null) { + return DefaultMetricsSystem.instance().register( + METRICS_SOURCE_NAME, + "Metrics for component version management.", + new ComponentVersionManagerMetrics(versionManager)); + } + return metrics; + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + MetricsRecordBuilder builder = collector.addRecord(METRICS_SOURCE_NAME); + builder + .addGauge(SOFTWARE_VERSION, + versionManager.getSoftwareVersion().serialize()) + .addGauge(APPARENT_VERSION, + versionManager.getApparentVersion().serialize()); + } + + public void unRegister() { + DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME); + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java new file mode 100644 index 000000000000..1e76ff91be39 --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.upgrade; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.junit.jupiter.params.provider.Arguments; + +/** + * Tests for {@link HDDSVersionManager}. + */ +class TestHDDSVersionManager extends AbstractComponentVersionManagerTest { + + private static final List ALL_VERSIONS; + + static { + ALL_VERSIONS = new ArrayList<>(Arrays.asList(HDDSLayoutFeature.values())); + + for (HDDSVersion version : HDDSVersion.values()) { + // Add all defined versions after and including ZDU to get the complete version list. + if (HDDSVersion.ZDU.isSupportedBy(version) && version != HDDSVersion.FUTURE_VERSION) { + ALL_VERSIONS.add(version); + } + } + } + + public static Stream preFinalizedVersionArgs() { + return ALL_VERSIONS.stream() + .limit(ALL_VERSIONS.size() - 1) + .map(Arguments::of); + } + + @Override + protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { + return new HDDSVersionManager(serializedApparentVersion); + } + + @Override + protected List allVersionsInOrder() { + return ALL_VERSIONS; + } + + @Override + protected ComponentVersion expectedSoftwareVersion() { + return HDDSVersion.SOFTWARE_VERSION; + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java new file mode 100644 index 000000000000..0bf439326cac --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import static org.apache.ozone.test.MetricsAsserts.assertGauge; +import static org.apache.ozone.test.MetricsAsserts.getMetrics; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; + +/** + * Shared tests for concrete {@link ComponentVersionManager} implementations. + */ +public abstract class AbstractComponentVersionManagerTest { + + protected abstract ComponentVersionManager createManager(int serializedApparentVersion) throws IOException; + + protected abstract List allVersionsInOrder(); + + protected abstract ComponentVersion expectedSoftwareVersion(); + + @AfterEach + public void cleanupMetricsSource() { + DefaultMetricsSystem.instance().unregisterSource(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + } + + @Test + public void testApparentVersionTranslation() throws Exception { + for (ComponentVersion apparentVersion : allVersionsInOrder()) { + try (ComponentVersionManager versionManager = createManager(apparentVersion.serialize())) { + assertApparentVersion(versionManager, apparentVersion); + } + } + } + + @Test + public void testApparentVersionBehindSoftwareVersion() { + int serializedNextVersion = expectedSoftwareVersion().serialize() + 1; + assertThrows(IOException.class, () -> createManager(serializedNextVersion)); + } + + @ParameterizedTest + // Child classes must implement this as a static method to provide the versions to start finalization from. + @MethodSource("preFinalizedVersionArgs") + public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) throws Exception { + List allVersions = allVersionsInOrder(); + int apparentVersionIndex = allVersions.indexOf(apparentVersion); + assertTrue(apparentVersionIndex >= 0, "Apparent version " + apparentVersion + " must exist"); + Iterator expectedVersions = allVersions.subList(apparentVersionIndex + 1, allVersions.size()) + .iterator(); + + try (ComponentVersionManager versionManager = createManager(apparentVersion.serialize())) { + assertApparentVersion(versionManager, apparentVersion); + + for (ComponentVersion versionToFinalize : versionManager.getUnfinalizedVersions()) { + assertTrue(versionManager.needsFinalization()); + assertFalse(versionManager.isAllowed(versionToFinalize), + "Unfinalized version " + versionToFinalize + " should not be allowed by apparent version " + + versionManager.getApparentVersion()); + assertTrue(expectedVersions.hasNext()); + assertEquals(expectedVersions.next(), versionToFinalize); + + versionManager.markFinalized(versionToFinalize); + assertApparentVersion(versionManager, versionToFinalize); + } + + assertFalse(expectedVersions.hasNext()); + assertThrows(NoSuchElementException.class, expectedVersions::next); + } + } + + @Test + public void testFinalizationFromSoftwareVersionNoOp() throws Exception { + try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + versionManager.markFinalized(expectedSoftwareVersion()); + + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + } + + @Test + public void testFinalizationOfNonExistentVersion() throws Exception { + try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + + ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class); + when(mockVersion.isSupportedBy(any())).thenReturn(false); + + assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); + // The failed finalization call should not have changed the version manager's state. + assertApparentVersion(versionManager, expectedSoftwareVersion()); + assertFalse(versionManager.needsFinalization()); + assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + } + } + + private void assertApparentVersion(ComponentVersionManager versionManager, ComponentVersion apparentVersion) { + assertEquals(apparentVersion, versionManager.getApparentVersion()); + assertTrue(versionManager.isAllowed(apparentVersion), apparentVersion + " should be allowed"); + assertEquals(expectedSoftwareVersion(), versionManager.getSoftwareVersion(), + "Software version should never change"); + if (!versionManager.needsFinalization()) { + assertTrue(versionManager.isAllowed(expectedSoftwareVersion()), + "Software version should always be allowed when finalized"); + } + MetricsRecordBuilder metrics = getMetrics(ComponentVersionManagerMetrics.METRICS_SOURCE_NAME); + assertGauge("SoftwareVersion", expectedSoftwareVersion().serialize(), metrics); + assertGauge("ApparentVersion", apparentVersion.serialize(), metrics); + } +} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java index f006698518a8..6cd9391e28f7 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -29,6 +29,7 @@ import java.util.Iterator; import javax.management.MBeanServer; import javax.management.ObjectName; +import org.apache.hadoop.hdds.ComponentVersion; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -188,6 +189,12 @@ public int layoutVersion() { public String description() { return null; } + + @Override + public ComponentVersion nextVersion() { + // TODO HDDS-14826 will remove this test. No need to add handling for this new method. + return null; + } }; } return lfs; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java index ade86b488b9b..89a52b7c31d4 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestUpgradeFinalizerActions.java @@ -86,6 +86,13 @@ public String description() { return null; } + @Override + public MockLayoutFeature nextVersion() { + // TODO HDDS-14826 will remove the tests that are using this. No need to provide an implementation for this new + // method. + return null; + } + @Override public String toString() { return name() + " (" + serialize() + ")"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index c19b720682d4..9f69215b6943 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -17,11 +17,21 @@ package org.apache.hadoop.ozone.om.upgrade; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.upgrade.LayoutFeature; /** - * List of OM Layout features / versions. + * List of OM Layout Features. All version management has been migrated to {@link OzoneManagerVersion} and no new + * additions should be made to this class. Existing versions are kept here for backwards compatibility when upgrading + * to this version from older versions. */ public enum OMLayoutFeature implements LayoutFeature { ////////////////////////////// ////////////////////////////// @@ -46,8 +56,14 @@ public enum OMLayoutFeature implements LayoutFeature { DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"), SNAPSHOT_DEFRAG(9, "Supporting defragmentation of snapshot"); + // ALL NEW VERSIONS SHOULD NOW BE ADDED TO OzoneManagerVersion + /////////////////////////////// ///////////////////////////// + private static final SortedMap BY_VALUE = + Arrays.stream(values()) + .collect(toMap(OMLayoutFeature::serialize, identity(), (v1, v2) -> v1, TreeMap::new)); + private final int layoutVersion; private final String description; private OmUpgradeAction action; @@ -62,6 +78,15 @@ public int layoutVersion() { return layoutVersion; } + /** + * @param version The serialized version to convert. + * @return The version corresponding to this serialized value, or {@code null} if no matching version is + * found. + */ + public static OMLayoutFeature deserialize(int version) { + return BY_VALUE.get(version); + } + @Override public String description() { return description; @@ -84,6 +109,21 @@ public void addAction(OmUpgradeAction upgradeAction) { } } + /** + * @return The next version immediately following this one. If there is no next version found in this enum, + * the next version is {@link OzoneManagerVersion#ZDU}, since all OM versioning has been migrated to + * {@link OzoneManagerVersion} as part of the ZDU feature. + */ + @Override + public ComponentVersion nextVersion() { + OMLayoutFeature nextFeature = BY_VALUE.get(layoutVersion + 1); + if (nextFeature == null) { + return OzoneManagerVersion.ZDU; + } else { + return nextFeature; + } + } + @Override public Optional action() { return Optional.ofNullable(action); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java new file mode 100644 index 000000000000..f250928b2a97 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.io.IOException; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; + +/** + * Component version manager for Ozone Manager. + */ +public class OMVersionManager extends ComponentVersionManager { + public OMVersionManager(int serializedApparentVersion) throws IOException { + super(computeApparentVersion(serializedApparentVersion), OzoneManagerVersion.SOFTWARE_VERSION); + } + + /** + * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link OzoneManagerVersion}. + * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the + * ZDU feature, and the apparent version corresponds to a version in {@link OMLayoutFeature}. + */ + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { + if (serializedApparentVersion < OzoneManagerVersion.ZDU.serialize()) { + return OMLayoutFeature.deserialize(serializedApparentVersion); + } else { + return OzoneManagerVersion.deserialize(serializedApparentVersion); + } + } + + // TODO HDDS-14826: Register upgrade actions based on annotations +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java new file mode 100644 index 000000000000..9e98935b5b3d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeature.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertNotSupportedBy; +import static org.apache.hadoop.hdds.ComponentVersionTestUtils.assertSupportedBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.junit.jupiter.api.Test; + +/** + * Tests invariants for legacy OM layout feature versions. + */ +public class TestOMLayoutFeature { + @Test + public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() { + OMLayoutFeature[] values = OMLayoutFeature.values(); + int currVersion = -1; + for (OMLayoutFeature lf : values) { + // This will skip the jump from the last OMLayoutFeature to OzoneManagerVersion#ZDU, + // since that is expected to be a larger version increment. + assertEquals(currVersion + 1, lf.layoutVersion(), + "Expected monotonically increasing layout version for " + lf); + currVersion = lf.layoutVersion(); + } + } + + /** + * All incompatible changes to OM should now be added to {@link OzoneManagerVersion}. + */ + @Test + public void testNoNewOMLayoutFeaturesAdded() { + int numOMLayoutFeatures = OMLayoutFeature.values().length; + OMLayoutFeature lastFeature = OMLayoutFeature.values()[numOMLayoutFeatures - 1]; + assertEquals(10, numOMLayoutFeatures); + assertEquals(OMLayoutFeature.SNAPSHOT_DEFRAG, lastFeature); + assertEquals(9, lastFeature.layoutVersion()); + } + + @Test + public void testNextVersion() { + OMLayoutFeature[] values = OMLayoutFeature.values(); + for (int i = 1; i < values.length; i++) { + OMLayoutFeature previous = values[i - 1]; + OMLayoutFeature current = values[i]; + assertEquals(current, previous.nextVersion(), + "Expected " + previous + ".nextVersion() to be " + current); + } + // The last layout feature should point us to the ZDU version to switch to using OzoneManagerVersion. + assertEquals(OzoneManagerVersion.ZDU, values[values.length - 1].nextVersion()); + } + + @Test + public void testSerDes() { + for (OMLayoutFeature version : OMLayoutFeature.values()) { + assertEquals(version, OMLayoutFeature.deserialize(version.serialize())); + } + } + + @Test + public void testDeserializeUnknownVersionReturnsNull() { + assertNull(OMLayoutFeature.deserialize(-1)); + assertNull(OMLayoutFeature.deserialize(Integer.MAX_VALUE)); + // OMLayoutFeature can only deserialize values from its own enum. + assertNull(OMLayoutFeature.deserialize(OzoneManagerVersion.ZDU.serialize())); + } + + @Test + public void testIsSupportedByFeatureBoundary() { + for (OMLayoutFeature feature : OMLayoutFeature.values()) { + // A layout feature should support itself. + int layoutVersion = feature.layoutVersion(); + assertSupportedBy(feature, feature); + if (layoutVersion > 0) { + // A layout feature should not be supported by older features. + OMLayoutFeature previousFeature = OMLayoutFeature.values()[layoutVersion - 1]; + assertNotSupportedBy(feature, previousFeature); + } + } + } + + @Test + public void testAllLayoutFeaturesAreSupportedByFutureVersions() { + for (OMLayoutFeature feature : OMLayoutFeature.values()) { + assertSupportedBy(feature, OzoneManagerVersion.ZDU); + assertSupportedBy(feature, OzoneManagerVersion.FUTURE_VERSION); + // No ComponentVersion instance represents an arbitrary unknown future version. + assertTrue(feature.isSupportedBy(Integer.MAX_VALUE)); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java new file mode 100644 index 000000000000..2c2044e9d04f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutVersionManager.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.OM_UPGRADE_CLASS_PACKAGE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Optional; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.junit.jupiter.api.Test; + +/** + * Test OM layout version management. + */ +public class TestOMLayoutVersionManager { + + @Test + public void testOMLayoutVersionManager() throws IOException { + OMLayoutVersionManager omVersionManager = + new OMLayoutVersionManager(); + + // Initial Version is always allowed. + assertTrue(omVersionManager.isAllowed(INITIAL_VERSION)); + assertThat(INITIAL_VERSION.layoutVersion()) + .isLessThanOrEqualTo(omVersionManager.getMetadataLayoutVersion()); + } + + @Test + public void testOMLayoutVersionManagerInitError() { + int lV = OMLayoutFeature.values()[OMLayoutFeature.values().length - 1] + .layoutVersion() + 1; + OMException ome = assertThrows(OMException.class, () -> new OMLayoutVersionManager(lV)); + assertEquals(NOT_SUPPORTED_OPERATION, ome.getResult()); + } + + @Test + + /* + * The OMLayoutFeatureAspect relies on the fact that the OM client + * request handler class has a preExecute method with first argument as + * 'OzoneManager'. If that is not true, please fix + * OMLayoutFeatureAspect#beforeRequestApplyTxn. + */ + public void testOmClientRequestPreExecuteIsCompatibleWithAspect() { + Method[] methods = OMClientRequest.class.getMethods(); + + Optional preExecuteMethod = Arrays.stream(methods) + .filter(m -> m.getName().equals("preExecute")) + .findFirst(); + + assertTrue(preExecuteMethod.isPresent()); + assertThat(preExecuteMethod.get().getParameterCount()).isGreaterThanOrEqualTo(1); + assertEquals(OzoneManager.class, + preExecuteMethod.get().getParameterTypes()[0]); + } + + @Test + public void testOmUpgradeActionsRegistered() throws Exception { + OMLayoutVersionManager lvm = new OMLayoutVersionManager(); // MLV >= 0 + assertFalse(lvm.needsFinalization()); + + // INITIAL_VERSION is finalized, hence should not register. + Optional action = + INITIAL_VERSION.action(); + assertFalse(action.isPresent()); + + lvm = mock(OMLayoutVersionManager.class); + when(lvm.getMetadataLayoutVersion()).thenReturn(-1); + doCallRealMethod().when(lvm).registerUpgradeActions(anyString()); + lvm.registerUpgradeActions(OM_UPGRADE_CLASS_PACKAGE); + + action = INITIAL_VERSION.action(); + assertTrue(action.isPresent()); + assertEquals(MockOmUpgradeAction.class, action.get().getClass()); + OzoneManager omMock = mock(OzoneManager.class); + action.get().execute(omMock); + verify(omMock, times(1)).getVersion(); + } + + /** + * Mock OM upgrade action class. + */ + @UpgradeActionOm(feature = + INITIAL_VERSION) + public static class MockOmUpgradeAction implements OmUpgradeAction { + + @Override + public void execute(OzoneManager arg) { + arg.getVersion(); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java index 388b5134c41b..eed52733acbb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -17,128 +17,52 @@ package org.apache.hadoop.ozone.om.upgrade; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; -import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; -import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.OM_UPGRADE_CLASS_PACKAGE; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doCallRealMethod; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import java.io.IOException; -import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Optional; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.om.request.OMClientRequest; -import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; +import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.junit.jupiter.params.provider.Arguments; /** - * Test OM layout version management. + * Tests for {@link OMVersionManager}. */ -public class TestOMVersionManager { - - @Test - public void testOMLayoutVersionManager() throws IOException { - OMLayoutVersionManager omVersionManager = - new OMLayoutVersionManager(); +class TestOMVersionManager extends AbstractComponentVersionManagerTest { - // Initial Version is always allowed. - assertTrue(omVersionManager.isAllowed(INITIAL_VERSION)); - assertThat(INITIAL_VERSION.layoutVersion()) - .isLessThanOrEqualTo(omVersionManager.getMetadataLayoutVersion()); - } + private static final List ALL_VERSIONS; - @Test - public void testOMLayoutVersionManagerInitError() { - int lV = OMLayoutFeature.values()[OMLayoutFeature.values().length - 1] - .layoutVersion() + 1; - OMException ome = assertThrows(OMException.class, () -> new OMLayoutVersionManager(lV)); - assertEquals(NOT_SUPPORTED_OPERATION, ome.getResult()); - } - - @Test - public void testOMLayoutFeaturesHaveIncreasingLayoutVersion() - throws Exception { - OMLayoutFeature[] values = OMLayoutFeature.values(); - int currVersion = -1; - OMLayoutFeature lastFeature = null; - for (OMLayoutFeature lf : values) { - assertEquals(currVersion + 1, lf.layoutVersion()); - currVersion = lf.layoutVersion(); - lastFeature = lf; + static { + ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values())); + for (OzoneManagerVersion version : OzoneManagerVersion.values()) { + // Add all defined versions after and including ZDU to get the complete version list. + if (OzoneManagerVersion.ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { + ALL_VERSIONS.add(version); + } } - lastFeature.addAction(arg -> { - String v = arg.getVersion(); - }); - - OzoneManager omMock = mock(OzoneManager.class); - lastFeature.action().get().execute(omMock); - verify(omMock, times(1)).getVersion(); } - @Test - - /* - * The OMLayoutFeatureAspect relies on the fact that the OM client - * request handler class has a preExecute method with first argument as - * 'OzoneManager'. If that is not true, please fix - * OMLayoutFeatureAspect#beforeRequestApplyTxn. - */ - public void testOmClientRequestPreExecuteIsCompatibleWithAspect() { - Method[] methods = OMClientRequest.class.getMethods(); - - Optional preExecuteMethod = Arrays.stream(methods) - .filter(m -> m.getName().equals("preExecute")) - .findFirst(); - - assertTrue(preExecuteMethod.isPresent()); - assertThat(preExecuteMethod.get().getParameterCount()).isGreaterThanOrEqualTo(1); - assertEquals(OzoneManager.class, - preExecuteMethod.get().getParameterTypes()[0]); + public static Stream preFinalizedVersionArgs() { + return ALL_VERSIONS.stream() + .limit(ALL_VERSIONS.size() - 1) + .map(Arguments::of); } - @Test - public void testOmUpgradeActionsRegistered() throws Exception { - OMLayoutVersionManager lvm = new OMLayoutVersionManager(); // MLV >= 0 - assertFalse(lvm.needsFinalization()); - - // INITIAL_VERSION is finalized, hence should not register. - Optional action = - INITIAL_VERSION.action(); - assertFalse(action.isPresent()); - - lvm = mock(OMLayoutVersionManager.class); - when(lvm.getMetadataLayoutVersion()).thenReturn(-1); - doCallRealMethod().when(lvm).registerUpgradeActions(anyString()); - lvm.registerUpgradeActions(OM_UPGRADE_CLASS_PACKAGE); - - action = INITIAL_VERSION.action(); - assertTrue(action.isPresent()); - assertEquals(MockOmUpgradeAction.class, action.get().getClass()); - OzoneManager omMock = mock(OzoneManager.class); - action.get().execute(omMock); - verify(omMock, times(1)).getVersion(); + @Override + protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { + return new OMVersionManager(serializedApparentVersion); } - /** - * Mock OM upgrade action class. - */ - @UpgradeActionOm(feature = - INITIAL_VERSION) - public static class MockOmUpgradeAction implements OmUpgradeAction { + @Override + protected List allVersionsInOrder() { + return ALL_VERSIONS; + } - @Override - public void execute(OzoneManager arg) { - arg.getVersion(); - } + @Override + protected ComponentVersion expectedSoftwareVersion() { + return OzoneManagerVersion.SOFTWARE_VERSION; } }