Optionally configure crdb listen port in dev.

At the moment, omicron-dev starts crdb on port 0, which tells the database to
use any available port. This is fine for most use cases, but less fine for
running a second nexus alongside an existing run-all:
https://github.com/oxidecomputer/omicron/blob/main/docs/how-to-run-simulated.adoc#using-both-omicron-dev-run-all-and-running-nexus-manually.
This process involves copying the randomly selected ports from the output of
run-all and writing them to a custom nexus config. To make the process simpler,
this patch allows the user to configure the crdb listen port, defaulting to the
current choice of 0.

Author: jmcarp

dev-tools/omicron-dev/Cargo.toml

@@ -22,6 +22,7 @@ nexus-config.workspace = true
 nexus-test-interface.workspace = true
 nexus-test-utils = { workspace = true, features = ["omicron-dev"] }
 omicron-nexus.workspace = true
+omicron-test-utils.workspace = true
 omicron-workspace-hack.workspace = true
 oxide-tokio-rt.workspace = true
 # See omicron-rpaths for more about the "pq-sys" dependency.

dev-tools/omicron-dev/src/main.rs

@@ -11,6 +11,7 @@ use libc::SIGINT;
 use nexus_config::NexusConfig;
 use nexus_test_interface::NexusServer;
 use nexus_test_utils::resource_helpers::DiskTest;
+use omicron_test_utils::COCKROACHDB_DEFAULT_LISTEN_PORT;
 use signal_hook_tokio::Signals;
 use std::fs;
 
@@ -51,6 +52,9 @@ struct RunAllArgs {
     /// Nexus external API listen port.  Use `0` to request any available port.
     #[clap(long, action)]
     nexus_listen_port: Option<u16>,
+    /// CockroachDB listen port. Use `0` to request any available port.
+    #[clap(long, default_value_t = COCKROACHDB_DEFAULT_LISTEN_PORT)]
+    db_listen_port: u16,
     /// Override the gateway server configuration file.
     #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)]
     gateway_config: Utf8PathBuf,
@@ -88,11 +92,14 @@ impl RunAllArgs {
         }
 
         println!("omicron-dev: setting up all services ... ");
-        let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::<
-            omicron_nexus::Server,
-        >(&mut config, 0, self.gateway_config.clone())
-        .await
-        .context("error setting up services")?;
+        let cptestctx =
+            nexus_test_utils::omicron_dev_setup_with_config::<
+                omicron_nexus::Server,
+            >(
+                &mut config, 0, self.gateway_config.clone(), self.db_listen_port
+            )
+            .await
+            .context("error setting up services")?;
 
         println!("omicron-dev: Adding disks to first sled agent");
 

nexus/db-queries/src/db/pub_test_utils/crdb.rs

@@ -35,6 +35,7 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance {
     dev::test_setup_database(
         log,
         dev::StorageSource::CopyFromSeed { input_tar },
+        dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT,
     )
     .await
 }
@@ -45,18 +46,25 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance {
 pub async fn test_setup_database_empty(
     log: &Logger,
 ) -> dev::db::CockroachInstance {
-    dev::test_setup_database(log, dev::StorageSource::DoNotPopulate).await
+    dev::test_setup_database(
+        log,
+        dev::StorageSource::DoNotPopulate,
+        dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT,
+    )
+    .await
 }
 
 /// Wrapper around [`dev::test_setup_database`] which uses a seed tarball
 /// provided as an argument.
 pub async fn test_setup_database_from_seed(
     log: &Logger,
     input_tar: Utf8PathBuf,
+    listen_port: u16,
 ) -> dev::db::CockroachInstance {
     dev::test_setup_database(
         log,
         dev::StorageSource::CopyFromSeed { input_tar },
+        listen_port,
     )
     .await
 }

nexus/test-utils/src/nexus_test.rs

@@ -361,6 +361,7 @@ pub async fn omicron_dev_setup_with_config<N: NexusServer>(
     config: &mut NexusConfig,
     extra_sled_agents: u16,
     gateway_config_file: Utf8PathBuf,
+    listen_port: u16,
 ) -> Result<ControlPlaneTestContext<N>> {
     let starter = ControlPlaneStarter::<N>::new("omicron-dev", config);
 
@@ -382,7 +383,7 @@ pub async fn omicron_dev_setup_with_config<N: NexusServer>(
 
     Ok(setup_with_config_impl(
         starter,
-        PopulateCrdb::FromSeed { input_tar: seed_tar },
+        PopulateCrdb::FromSeed { input_tar: seed_tar, listen_port },
         sim::SimMode::Auto,
         None,
         extra_sled_agents,

nexus/test-utils/src/starter.rs

@@ -265,8 +265,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> {
                 crdb::test_setup_database(log).await
             }
             #[cfg(feature = "omicron-dev")]
-            PopulateCrdb::FromSeed { input_tar } => {
-                crdb::test_setup_database_from_seed(log, input_tar).await
+            PopulateCrdb::FromSeed { input_tar, listen_port } => {
+                crdb::test_setup_database_from_seed(log, input_tar, listen_port)
+                    .await
             }
             PopulateCrdb::Empty => crdb::test_setup_database_empty(log).await,
         };
@@ -1829,7 +1830,7 @@ pub(crate) enum PopulateCrdb {
 
     /// Populate Cockroach from the seed located at this path.
     #[cfg(feature = "omicron-dev")]
-    FromSeed { input_tar: camino::Utf8PathBuf },
+    FromSeed { input_tar: camino::Utf8PathBuf, listen_port: u16 },
 
     /// Do not populate Cockroach.
     Empty,

test-utils/src/dev/db.rs

@@ -33,7 +33,7 @@ const COCKROACHDB_START_TIMEOUT_DEFAULT: Duration = Duration::from_secs(30);
 // This is appropriate for the test suite and may be useful in some cases for
 // omicron-dev.  However, omicron-dev by default chooses a specific port so that
 // we can ship a Nexus configuration that will use the same port.
-const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0;
+pub const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0;
 
 /// CockroachDB database name
 // This MUST be kept in sync with dbinit.sql and dbwipe.sql.

test-utils/src/dev/mod.rs

@@ -62,18 +62,21 @@ pub enum StorageSource {
 pub async fn test_setup_database(
     log: &Logger,
     source: StorageSource,
+    listen_port: u16,
 ) -> db::CockroachInstance {
     usdt::register_probes().expect("Failed to register USDT DTrace probes");
-    setup_database(log, source).await.unwrap()
+    setup_database(log, source, listen_port).await.unwrap()
 }
 
 // TODO: switch to anyhow entirely -- this function is currently a mishmash of
 // anyhow and unwrap/expect calls.
 async fn setup_database(
     log: &Logger,
     storage_source: StorageSource,
+    listen_port: u16,
 ) -> Result<db::CockroachInstance> {
     let builder = db::CockroachStarterBuilder::new();
+    let builder = builder.listen_port(listen_port);
     let builder = match &storage_source {
         StorageSource::DoNotPopulate | StorageSource::CopyFromSeed { .. } => {
             builder

test-utils/src/dev/seed.rs

@@ -173,6 +173,7 @@ pub async fn test_setup_database_seed(
         super::StorageSource::PopulateLatest {
             output_dir: tmp_seed_dir.path().to_owned(),
         },
+        super::db::COCKROACHDB_DEFAULT_LISTEN_PORT,
     )
     .await
     .context("failed to setup database")?;

test-utils/src/lib.rs

@@ -15,6 +15,8 @@ use headers::authorization::Credentials;
 pub mod certificates;
 pub mod dev;
 
+pub use dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT;
+
 #[macro_use]
 extern crate slog;