diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 553da6f893f..a7831571d19 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -62,6 +62,10 @@ org.apache.ozone hdds-config + + org.apache.ozone + hdds-container-service + org.apache.ozone ozone-common @@ -78,6 +82,10 @@ org.reflections reflections + + org.slf4j + slf4j-api + org.kohsuke.metainf-services @@ -106,12 +114,6 @@ ratis-thirdparty-misc runtime - - - org.slf4j - slf4j-api - runtime - org.slf4j slf4j-reload4j diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java new file mode 100644 index 00000000000..cc1bab82c48 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_CLIENT_ADDRESS_KEY; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_CLIENT_BIND_HOST_KEY; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HTTP_BIND_HOST_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_ADMIN_PORT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_PORT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_PORT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_SERVER_PORT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.ozone.HddsDatanodeService; +import org.apache.hadoop.ozone.container.replication.ReplicationServer; +import org.apache.hadoop.ozone.local.LocalOzoneClusterConfig.FormatMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Starts a local in-process Ozone cluster for development and testing. + * + *

This implementation manages the lifecycle of datanodes (and eventually + * SCM, OM, S3G) running within a single JVM process.

+ */ +public final class LocalOzoneCluster implements LocalOzoneRuntime { + + private static final Logger LOG = + LoggerFactory.getLogger(LocalOzoneCluster.class); + + private static final String[] NO_ARGS = new String[0]; + private static final String PORTS_STATE_FILE = "ports.properties"; + + private final LocalOzoneClusterConfig config; + private final OzoneConfiguration seedConfiguration; + private final AtomicBoolean closed = new AtomicBoolean(); + private final List datanodes = new ArrayList<>(); + + private boolean previousMetricsMiniClusterMode; + private boolean metricsMiniClusterModeEnabled; + + /** + * Creates a new local Ozone cluster. + * + * @param config the cluster configuration + * @param seedConfiguration the base Ozone configuration + */ + public LocalOzoneCluster(LocalOzoneClusterConfig config, + OzoneConfiguration seedConfiguration) { + this.config = Objects.requireNonNull(config, "config"); + this.seedConfiguration = new OzoneConfiguration( + Objects.requireNonNull(seedConfiguration, "seedConfiguration")); + } + + @Override + public void start() throws Exception { + enableMiniClusterMetricsMode(); + prepareDataDirectory(); + + OzoneConfiguration baseConf = prepareBaseConfiguration(); + startDatanodes(baseConf); + + // TODO: Add SCM/OM startup and waitForClusterToBeReady() in future tasks + LOG.info("Local Ozone cluster started with {} datanode(s)", + datanodes.size()); + } + + @Override + public String getDisplayHost() { + return "0.0.0.0".equals(config.getHost()) + ? LocalOzoneClusterConfig.DEFAULT_HOST + : config.getHost(); + } + + /** + * Returns the number of running datanodes. + */ + public int getDatanodeCount() { + return datanodes.size(); + } + + @Override + public int getScmPort() { + // TODO: Return actual SCM port when SCM is implemented + return -1; + } + + @Override + public int getOmPort() { + // TODO: Return actual OM port when OM is implemented + return -1; + } + + @Override + public int getS3gPort() { + // TODO: Return actual S3G port when S3G is implemented + return -1; + } + + @Override + public String getS3Endpoint() { + // TODO: Return actual S3 endpoint when S3G is implemented + return ""; + } + + @Override + public void close() throws Exception { + if (!closed.compareAndSet(false, true)) { + return; + } + + try { + stopDatanodes(); + + if (config.isEphemeral()) { + deleteDirectory(config.getDataDir()); + LOG.info("Deleted ephemeral data directory: {}", config.getDataDir()); + } + } finally { + restoreMetricsMode(); + } + } + + /** + * Prepares the data directory, formatting if needed. + */ + private void prepareDataDirectory() throws IOException { + if (config.getFormatMode() == FormatMode.ALWAYS) { + deleteDirectory(config.getDataDir()); + } + Files.createDirectories(config.getDataDir()); + } + + /** + * Prepares the base Ozone configuration with local-safe defaults. + */ + OzoneConfiguration prepareBaseConfiguration() throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(seedConfiguration); + + // Local-safe replication defaults: single replica, no Ratis + conf.set(OZONE_REPLICATION, ReplicationFactor.ONE.name()); + conf.set(OZONE_REPLICATION_TYPE, ReplicationType.STAND_ALONE.name()); + conf.setBoolean(HDDS_CONTAINER_RATIS_ENABLED_KEY, false); + + // Root metadata directory + Path metadataDir = Files.createDirectories( + config.getDataDir().resolve("metadata")); + conf.set(OZONE_METADATA_DIRS, metadataDir.toString()); + + return conf; + } + + /** + * Starts the configured number of datanodes. + */ + private void startDatanodes(OzoneConfiguration baseConf) throws IOException { + PersistedPorts persistedPorts = PersistedPorts.load( + config.getDataDir().resolve(PORTS_STATE_FILE)); + PortAllocator ports = new PortAllocator(); + + for (int index = 0; index < config.getDatanodes(); index++) { + OzoneConfiguration dnConf = createDatanodeConfiguration( + baseConf, index, ports, persistedPorts); + HddsDatanodeService datanode = startDatanode(dnConf); + datanodes.add(datanode); + LOG.info("Started datanode {} of {}", index + 1, config.getDatanodes()); + } + + persistedPorts.store(); + } + + /** + * Creates isolated configuration for a single datanode. + */ + private OzoneConfiguration createDatanodeConfiguration( + OzoneConfiguration baseConf, int index, PortAllocator ports, + PersistedPorts persistedPorts) throws IOException { + + OzoneConfiguration dnConf = new OzoneConfiguration(baseConf); + + // Create isolated directories for this datanode + Path datanodeDir = Files.createDirectories( + config.getDataDir().resolve("datanode-" + (index + 1))); + Path metaDir = Files.createDirectories(datanodeDir.resolve("metadata")); + Path dataDir = Files.createDirectories(datanodeDir.resolve("data-0")); + Path ratisDir = Files.createDirectories(datanodeDir.resolve("ratis")); + + dnConf.set(OZONE_METADATA_DIRS, metaDir.toString()); + dnConf.set(HDDS_DATANODE_DIR_KEY, dataDir.toString()); + dnConf.set(HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, ratisDir.toString()); + + // Allocate unique ports for this datanode + String prefix = "dn." + index; + + dnConf.set(HDDS_DATANODE_HTTP_ADDRESS_KEY, + address(config.getHost(), reservePort(ports, persistedPorts, + prefix + ".http"))); + dnConf.set(HDDS_DATANODE_HTTP_BIND_HOST_KEY, config.getBindHost()); + + dnConf.set(HDDS_DATANODE_CLIENT_ADDRESS_KEY, + address(config.getHost(), reservePort(ports, persistedPorts, + prefix + ".client"))); + dnConf.set(HDDS_DATANODE_CLIENT_BIND_HOST_KEY, config.getBindHost()); + + dnConf.setInt(HDDS_CONTAINER_IPC_PORT, + reservePort(ports, persistedPorts, prefix + ".container.ipc")); + dnConf.setInt(HDDS_CONTAINER_RATIS_IPC_PORT, + reservePort(ports, persistedPorts, prefix + ".ratis.ipc")); + dnConf.setInt(HDDS_CONTAINER_RATIS_ADMIN_PORT, + reservePort(ports, persistedPorts, prefix + ".ratis.admin")); + dnConf.setInt(HDDS_CONTAINER_RATIS_SERVER_PORT, + reservePort(ports, persistedPorts, prefix + ".ratis.server")); + dnConf.setInt(HDDS_CONTAINER_RATIS_DATASTREAM_PORT, + reservePort(ports, persistedPorts, prefix + ".ratis.datastream")); + + dnConf.setFromObject(new ReplicationServer.ReplicationConfig() + .setPort(reservePort(ports, persistedPorts, prefix + ".replication"))); + + return dnConf; + } + + /** + * Starts a single datanode with the given configuration. + */ + private HddsDatanodeService startDatanode(OzoneConfiguration conf) + throws IOException { + HddsDatanodeService datanode = new HddsDatanodeService(NO_ARGS); + datanode.setConfiguration(conf); + datanode.start(conf); + return datanode; + } + + /** + * Stops all running datanodes in reverse order. + */ + private void stopDatanodes() { + for (int i = datanodes.size() - 1; i >= 0; i--) { + HddsDatanodeService datanode = datanodes.get(i); + try { + datanode.stop(); + datanode.join(); + LOG.info("Stopped datanode {}", i + 1); + } catch (Exception ex) { + LOG.warn("Failed to stop datanode {}", i + 1, ex); + } + } + datanodes.clear(); + } + + /** + * Reserves a port, preferring a previously persisted value. + */ + private int reservePort(PortAllocator allocator, + PersistedPorts persistedPorts, String key) throws IOException { + int preferredPort = persistedPorts.get(key); + int port = allocator.reserve(preferredPort); + persistedPorts.set(key, port); + return port; + } + + /** + * Enables mini-cluster mode for metrics to allow multiple services + * in the same JVM. + */ + private void enableMiniClusterMetricsMode() { + if (!metricsMiniClusterModeEnabled) { + previousMetricsMiniClusterMode = DefaultMetricsSystem.inMiniClusterMode(); + DefaultMetricsSystem.setMiniClusterMode(true); + metricsMiniClusterModeEnabled = true; + } + } + + /** + * Restores the previous metrics mode. + */ + private void restoreMetricsMode() { + if (metricsMiniClusterModeEnabled) { + DefaultMetricsSystem.setMiniClusterMode(previousMetricsMiniClusterMode); + metricsMiniClusterModeEnabled = false; + } + } + + private static String address(String host, int port) { + return host + ":" + port; + } + + private static void deleteDirectory(Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (Stream paths = Files.walk(directory)) { + for (Path path : (Iterable) paths + .sorted(Comparator.reverseOrder())::iterator) { + Files.deleteIfExists(path); + } + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PersistedPorts.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PersistedPorts.java new file mode 100644 index 00000000000..8e14cd5e259 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PersistedPorts.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +/** + * Persists allocated ports to enable stable endpoints across restarts. + * + *

When a local Ozone cluster starts, it allocates ephemeral ports for + * its services. This class saves those ports to a properties file so that + * subsequent restarts can reuse the same ports, providing stable endpoints + * for clients.

+ */ +final class PersistedPorts { + + private final Path path; + private final Properties properties = new Properties(); + + private PersistedPorts(Path path) { + this.path = path; + } + + /** + * Loads persisted ports from the specified file. + * + * @param path the path to the ports properties file + * @return a PersistedPorts instance, empty if file doesn't exist + * @throws IOException if reading the file fails + */ + static PersistedPorts load(Path path) throws IOException { + PersistedPorts persistedPorts = new PersistedPorts(path); + if (Files.exists(path)) { + try (InputStream input = Files.newInputStream(path)) { + persistedPorts.properties.load(input); + } + } + return persistedPorts; + } + + /** + * Gets a previously persisted port value. + * + * @param key the port identifier (e.g., "dn.0.container.ipc") + * @return the port number, or 0 if not persisted + */ + int get(String key) { + String value = properties.getProperty(key); + return value == null ? 0 : Integer.parseInt(value); + } + + /** + * Sets a port value to be persisted. + * + * @param key the port identifier + * @param port the port number + */ + void set(String key, int port) { + properties.setProperty(key, Integer.toString(port)); + } + + /** + * Saves all port values to the file. + * + * @throws IOException if writing the file fails + */ + void store() throws IOException { + try (OutputStream output = Files.newOutputStream(path)) { + properties.store(output, "Local Ozone reserved ports"); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PortAllocator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PortAllocator.java new file mode 100644 index 00000000000..822e6602e07 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/PortAllocator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.HashSet; +import java.util.Set; + +/** + * Allocates unique ports for local Ozone services. + * + *

This allocator ensures that each service gets a unique port, + * either by using a preferred port or by finding a free ephemeral port. + * It tracks all reserved ports to prevent conflicts when multiple + * services are started in the same JVM.

+ */ +final class PortAllocator { + + private final Set reserved = new HashSet<>(); + + /** + * Reserves a port for use by a local service. + * + * @param preferredPort the preferred port to use, or 0 to auto-allocate + * @return the reserved port number + * @throws IOException if the preferred port is already reserved or + * if no free port can be found + */ + int reserve(int preferredPort) throws IOException { + if (preferredPort > 0) { + if (!reserved.add(preferredPort)) { + throw new IOException("Port " + preferredPort + + " is configured more than once."); + } + return preferredPort; + } + + while (true) { + int candidate = findFreePort(); + if (reserved.add(candidate)) { + return candidate; + } + } + } + + /** + * Finds a free ephemeral port by opening and immediately closing + * a server socket. + */ + private static int findFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(false); + return socket.getLocalPort(); + } + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java new file mode 100644 index 00000000000..ae5f6591543 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Unit tests for {@link LocalOzoneCluster}. + * + *

These tests focus on configuration generation without starting + * actual services, which require SCM.

+ */ +class TestLocalOzoneCluster { + + @TempDir + private Path tempDir; + + @Test + void prepareBaseConfigurationSetsReplicationDefaults() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + OzoneConfiguration baseConf = cluster.prepareBaseConfiguration(); + + assertEquals(ReplicationFactor.ONE.name(), + baseConf.get(OZONE_REPLICATION)); + assertEquals(ReplicationType.STAND_ALONE.name(), + baseConf.get(OZONE_REPLICATION_TYPE)); + assertFalse(baseConf.getBoolean(HDDS_CONTAINER_RATIS_ENABLED_KEY, true)); + } + + @Test + void prepareBaseConfigurationCreatesMetadataDir() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + OzoneConfiguration baseConf = cluster.prepareBaseConfiguration(); + + String metadataDir = baseConf.get(OZONE_METADATA_DIRS); + assertTrue(Files.exists(Paths.get(metadataDir)), + "Metadata directory should be created"); + assertTrue(metadataDir.contains("metadata"), + "Metadata dir path should contain 'metadata'"); + } + + @Test + void getDisplayHostReturnsConfiguredHost() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .setHost("192.168.1.100") + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + assertEquals("192.168.1.100", cluster.getDisplayHost()); + } + + @Test + void getDisplayHostReturnsLocalhostForBindAll() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .setHost("0.0.0.0") + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + assertEquals("127.0.0.1", cluster.getDisplayHost()); + } + + @Test + void getDatanodeCountReturnsZeroBeforeStart() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .setDatanodes(3) + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + assertEquals(0, cluster.getDatanodeCount(), + "Should have zero datanodes before start"); + } + + @Test + void closeIsIdempotent() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(tempDir) + .build(); + + LocalOzoneCluster cluster = new LocalOzoneCluster(config, + new OzoneConfiguration()); + + // Multiple closes should not throw + cluster.close(); + cluster.close(); + cluster.close(); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPersistedPorts.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPersistedPorts.java new file mode 100644 index 00000000000..ee00128bbc2 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPersistedPorts.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Unit tests for {@link PersistedPorts}. + */ +class TestPersistedPorts { + + @TempDir + private Path tempDir; + + @Test + void loadFromNonExistentFileReturnsEmpty() throws IOException { + Path portsFile = tempDir.resolve("ports.properties"); + PersistedPorts ports = PersistedPorts.load(portsFile); + + assertEquals(0, ports.get("nonexistent.key")); + } + + @Test + void setAndGetReturnsValue() throws IOException { + Path portsFile = tempDir.resolve("ports.properties"); + PersistedPorts ports = PersistedPorts.load(portsFile); + + ports.set("dn.0.http", 9878); + assertEquals(9878, ports.get("dn.0.http")); + } + + @Test + void storeAndLoadPreservesValues() throws IOException { + Path portsFile = tempDir.resolve("ports.properties"); + + // Store some ports + PersistedPorts ports1 = PersistedPorts.load(portsFile); + ports1.set("dn.0.http", 9878); + ports1.set("dn.0.client", 9879); + ports1.set("dn.1.http", 9880); + ports1.store(); + + assertTrue(Files.exists(portsFile), "Ports file should be created"); + + // Load and verify + PersistedPorts ports2 = PersistedPorts.load(portsFile); + assertEquals(9878, ports2.get("dn.0.http")); + assertEquals(9879, ports2.get("dn.0.client")); + assertEquals(9880, ports2.get("dn.1.http")); + assertEquals(0, ports2.get("nonexistent")); + } + + @Test + void storeOverwritesExistingFile() throws IOException { + Path portsFile = tempDir.resolve("ports.properties"); + + // First store + PersistedPorts ports1 = PersistedPorts.load(portsFile); + ports1.set("key1", 1111); + ports1.store(); + + // Second store with different value + PersistedPorts ports2 = PersistedPorts.load(portsFile); + ports2.set("key1", 2222); + ports2.set("key2", 3333); + ports2.store(); + + // Verify final state + PersistedPorts ports3 = PersistedPorts.load(portsFile); + assertEquals(2222, ports3.get("key1")); + assertEquals(3333, ports3.get("key2")); + } + + @Test + void loadPreservesExistingValuesWhenAddingNew() throws IOException { + Path portsFile = tempDir.resolve("ports.properties"); + + // Store initial value + PersistedPorts ports1 = PersistedPorts.load(portsFile); + ports1.set("existing", 1111); + ports1.store(); + + // Load, add new, store + PersistedPorts ports2 = PersistedPorts.load(portsFile); + assertEquals(1111, ports2.get("existing")); + ports2.set("new", 2222); + ports2.store(); + + // Verify both exist + PersistedPorts ports3 = PersistedPorts.load(portsFile); + assertEquals(1111, ports3.get("existing")); + assertEquals(2222, ports3.get("new")); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPortAllocator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPortAllocator.java new file mode 100644 index 00000000000..e1f85a5c8e6 --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestPortAllocator.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link PortAllocator}. + */ +class TestPortAllocator { + + @Test + void reserveWithPreferredPortReturnsPreferred() throws IOException { + PortAllocator allocator = new PortAllocator(); + int port = allocator.reserve(9878); + assertEquals(9878, port); + } + + @Test + void reserveWithZeroAllocatesEphemeralPort() throws IOException { + PortAllocator allocator = new PortAllocator(); + int port = allocator.reserve(0); + assertTrue(port > 0, "Should allocate a valid port"); + assertTrue(port <= 65535, "Port should be in valid range"); + } + + @Test + void reserveAllocatesUniqueEphemeralPorts() throws IOException { + PortAllocator allocator = new PortAllocator(); + Set ports = new HashSet<>(); + + for (int i = 0; i < 10; i++) { + int port = allocator.reserve(0); + assertTrue(ports.add(port), + "Port " + port + " was allocated more than once"); + } + } + + @Test + void reserveRejectsDuplicatePreferredPort() throws IOException { + PortAllocator allocator = new PortAllocator(); + allocator.reserve(9878); + + IOException exception = assertThrows(IOException.class, + () -> allocator.reserve(9878)); + assertTrue(exception.getMessage().contains("9878")); + assertTrue(exception.getMessage().contains("more than once")); + } + + @Test + void multipleAllocatorsAreIndependent() throws IOException { + PortAllocator allocator1 = new PortAllocator(); + PortAllocator allocator2 = new PortAllocator(); + + // Both should be able to reserve the same preferred port + // (they track independently, actual port conflict would happen at bind time) + int port1 = allocator1.reserve(9878); + int port2 = allocator2.reserve(9878); + + assertEquals(port1, port2); + } + + @Test + void ephemeralPortsAreDifferentEachTime() throws IOException { + PortAllocator allocator1 = new PortAllocator(); + PortAllocator allocator2 = new PortAllocator(); + + int port1 = allocator1.reserve(0); + int port2 = allocator2.reserve(0); + + // While not guaranteed, ephemeral ports should generally be different + // This test may occasionally fail but validates the allocation mechanism + assertNotEquals(port1, port2, + "Ephemeral ports from different allocators should typically differ"); + } +}