diff --git a/crates/smoketests/DEVELOP.md b/crates/smoketests/DEVELOP.md index 423c038f6ec..6acba49aea2 100644 --- a/crates/smoketests/DEVELOP.md +++ b/crates/smoketests/DEVELOP.md @@ -18,6 +18,26 @@ cargo smoketest test_sql_format cargo smoketest "cli::" # Run all CLI tests ``` +### Remote Servers + +Run against a standalone-compatible remote server with: + +```bash +cargo ci smoketests --server https://example.spacetimedb.com +``` + +Maincloud and maincloud staging require SpacetimeAuth-issued tokens rather than +server-issued tokens. Use `--auth-host` for those: + +```bash +cargo ci smoketests --server https://maincloud.staging.spacetimedb.com --auth-host +``` + +The runner invokes `spacetime login` once, then copies that logged-in config +into each isolated smoketest config. Tests that need throwaway server-issued +identities should call `require_server_issued_login!()` so they skip in +SpacetimeAuth mode. + ### WARNING: Stale Binary Risk **Smoketests use pre-built binaries and DO NOT automatically rebuild them.** diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index aaca9c59df8..225c2864b3f 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -78,6 +78,11 @@ pub fn is_remote_server() -> bool { remote_server_url().is_some() } +/// Returns true if remote smoketests are using a SpacetimeAuth-issued token. +pub fn is_using_auth_host() -> bool { + std::env::var("SPACETIME_USE_AUTH_HOST").ok().as_deref() == Some("1") +} + /// Skip this test if running against a remote server. /// /// Use this macro at the start of tests that require a local server, @@ -107,6 +112,19 @@ macro_rules! require_local_server { }; } +#[macro_export] +macro_rules! require_server_issued_login { + () => { + if $crate::is_using_auth_host() { + #[allow(clippy::disallowed_macros)] + { + eprintln!("Skipping test: requires server-issued throwaway identities"); + } + return; + } + }; +} + #[macro_export] macro_rules! require_dotnet { () => { @@ -501,7 +519,7 @@ pub struct PublishBuilder<'a> { break_clients: bool, num_replicas: Option, organization: Option, - force: bool, + force: Option<&'static str>, stdin_input: Option, source: Option, } @@ -528,7 +546,7 @@ impl<'a> PublishBuilder<'a> { break_clients: false, num_replicas: None, organization: None, - force: true, + force: Some("all"), stdin_input: None, source: None, } @@ -559,13 +577,13 @@ impl<'a> PublishBuilder<'a> { self } - pub fn force(mut self, force: bool) -> Self { + pub fn force(mut self, force: Option<&'static str>) -> Self { self.force = force; self } pub fn stdin(mut self, stdin_input: impl Into) -> Self { - self.force = false; + self.force = None; self.stdin_input = Some(stdin_input.into()); self } @@ -912,6 +930,10 @@ impl SmoketestBuilder { let module_name = format!("smoketest_module_{}", random_string()); let config_path = project_dir.path().join("config.toml"); + if let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") { + fs::copy(&base_config_path, &config_path) + .unwrap_or_else(|err| panic!("failed to copy base smoketest config from {base_config_path}: {err:#}")); + } let mut smoketest = Smoketest { guard, _data_dir_fixture: data_dir_fixture, @@ -1416,7 +1438,7 @@ log = "0.4" break_clients: bool, num_replicas: Option, organization: Option<&str>, - force: bool, + force: Option<&str>, stdin_input: Option<&str>, ) -> Result { let start = Instant::now(); @@ -1459,9 +1481,10 @@ log = "0.4" // Now publish with --bin-path to skip rebuild let publish_start = Instant::now(); let mut args = vec!["publish", "--server", &self.server_url, "--bin-path", &wasm_path_str]; - - if force { - args.push("--yes"); + let force_arg; + if let Some(force) = force { + force_arg = format!("--yes={force}"); + args.push(&force_arg); } if clear { diff --git a/crates/smoketests/tests/smoketests/cli/publish.rs b/crates/smoketests/tests/smoketests/cli/publish.rs index 50b32ce3e80..7fe5fab5fc0 100644 --- a/crates/smoketests/tests/smoketests/cli/publish.rs +++ b/crates/smoketests/tests/smoketests/cli/publish.rs @@ -16,12 +16,30 @@ fn cli_can_publish_spacetimedb_on_disk() { let dir = dir.to_string(); let _ = test - .spacetime(&["publish", "--module-path", &dir, "--server", &test.server_url, "foobar"]) + .spacetime(&[ + "publish", + "--module-path", + &dir, + "--server", + &test.server_url, + // Needed when running smoketests against a remote server. + "--yes=remote", + "foobar", + ]) .unwrap(); // Can republish without error to the same name let _ = test - .spacetime(&["publish", "--module-path", &dir, "--server", &test.server_url, "foobar"]) + .spacetime(&[ + "publish", + "--module-path", + &dir, + "--server", + &test.server_url, + // Needed when running smoketests against a remote server. + "--yes=remote", + "foobar", + ]) .unwrap(); } @@ -232,6 +250,8 @@ fn cli_publish_with_config_but_no_match_uses_cli_args() { "publish", "--server", &test.server_url, + // Needed when running smoketests against a remote server. + "--yes=remote", "cli-db-name", "--module-path", module_dir.to_str().unwrap(), diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 2d8c28b2f1f..c51ae66b0a7 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -1,6 +1,6 @@ use regex::Regex; use spacetimedb_smoketests::{ - require_dotnet, require_emscripten, require_pnpm, workspace_root, ModuleLanguage, Smoketest, + random_string, require_dotnet, require_emscripten, require_pnpm, workspace_root, ModuleLanguage, Smoketest, }; use std::{fs, path::Path}; @@ -1109,9 +1109,10 @@ fn cpp_http_test(name: &str, module_code: &str) -> (Smoketest, String) { fn typescript_http_test(name: &str, module_code: &str) -> (Smoketest, String) { require_pnpm!(); let mut test = Smoketest::builder().autopublish(false).build(); + let database_name = format!("{name}-{}", random_string()); let identity = test .publish() - .name(name) + .name(&database_name) .source(ModuleLanguage::TypeScript, name, module_code) .run() .unwrap(); diff --git a/crates/smoketests/tests/smoketests/new_user_flow.rs b/crates/smoketests/tests/smoketests/new_user_flow.rs index 21d7a7abdd8..99613e527a5 100644 --- a/crates/smoketests/tests/smoketests/new_user_flow.rs +++ b/crates/smoketests/tests/smoketests/new_user_flow.rs @@ -1,9 +1,13 @@ -use spacetimedb_smoketests::Smoketest; +use spacetimedb_smoketests::{require_server_issued_login, Smoketest}; // TODO: This test originally was testing to make sure that our tutorial isn't broken. Since our onboarding has changed we should probably update this test in the future. /// Test the entirety of the new user flow. #[test] fn test_new_user_flow() { + // This flow creates a throwaway server-issued identity with `new_identity`, + // which is not available when smoketests use SpacetimeAuth login. + require_server_issued_login!(); + let mut test = Smoketest::builder() .precompiled_module("new-user-flow") .autopublish(false) diff --git a/crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs b/crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs index 4e39555b91d..127860003a8 100644 --- a/crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs +++ b/crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs @@ -36,14 +36,22 @@ fn upgrade_prompt_on_publish() { let deny_err = test .publish() .name(&db_name) - .force(false) + // Needed when running smoketests against a remote server. + .force(Some("remote")) .run() .unwrap_err() .to_string(); assert!(deny_err.contains("major version upgrade from 1.0 to 2.0")); assert!(deny_err.contains("Please type 'upgrade' to accept this change:")); - let accepted_identity = test.publish().name(&db_name).stdin("upgrade\n").run().unwrap(); + let accepted_identity = test + .publish() + .name(&db_name) + .stdin("upgrade\n") + // Needed when running smoketests against a remote server. + .force(Some("remote")) + .run() + .unwrap(); assert_eq!(accepted_identity, initial_identity); } diff --git a/crates/smoketests/tests/smoketests/quickstart.rs b/crates/smoketests/tests/smoketests/quickstart.rs index cfa4f88400e..a6700f5ff06 100644 --- a/crates/smoketests/tests/smoketests/quickstart.rs +++ b/crates/smoketests/tests/smoketests/quickstart.rs @@ -766,7 +766,11 @@ log = "0.4" // Replace server address let host = self.test.server_host(); - let protocol = "http"; // The smoketest server uses http + let protocol = if self.test.server_url.starts_with("https://") { + "https" + } else { + "http" + }; main_code = main_code.replace("http://localhost:3000", &format!("{}://{}", protocol, host)); // Write the client code diff --git a/crates/smoketests/tests/smoketests/servers.rs b/crates/smoketests/tests/smoketests/servers.rs index 6fa3946da2f..c275721103f 100644 --- a/crates/smoketests/tests/smoketests/servers.rs +++ b/crates/smoketests/tests/smoketests/servers.rs @@ -1,9 +1,13 @@ use regex::Regex; -use spacetimedb_smoketests::Smoketest; +use spacetimedb_smoketests::{require_local_server, Smoketest}; /// Verify that we can add and list server configurations #[test] fn test_servers() { + // This only covers local CLI config behavior, so it is not valuable to run + // against remote servers. + require_local_server!(); + let test = Smoketest::builder().autopublish(false).build(); // Add a test server (local-only command, no --server flag needed) diff --git a/crates/smoketests/tests/smoketests/timestamp_route.rs b/crates/smoketests/tests/smoketests/timestamp_route.rs index 1d337bed1f0..d106e1a4cbb 100644 --- a/crates/smoketests/tests/smoketests/timestamp_route.rs +++ b/crates/smoketests/tests/smoketests/timestamp_route.rs @@ -1,10 +1,14 @@ -use spacetimedb_smoketests::{random_string, Smoketest}; +use spacetimedb_smoketests::{random_string, require_server_issued_login, Smoketest}; const TIMESTAMP_TAG: &str = "__timestamp_micros_since_unix_epoch__"; /// Test the /v1/database/{name}/unstable/timestamp endpoint #[test] fn test_timestamp_route() { + // This test creates a throwaway server-issued identity before publishing, + // which is not available when smoketests use SpacetimeAuth login. + require_server_issued_login!(); + let mut test = Smoketest::builder().autopublish(false).build(); let name = random_string(); diff --git a/crates/smoketests/tests/smoketests/views.rs b/crates/smoketests/tests/smoketests/views.rs index c3fd230a8ee..517242edba6 100644 --- a/crates/smoketests/tests/smoketests/views.rs +++ b/crates/smoketests/tests/smoketests/views.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use serde_json::{json, Value}; use spacetimedb_smoketests::{ - require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest, + random_string, require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest, }; const STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY: &str = @@ -881,8 +881,9 @@ fn test_procedure_triggers_subscription_updates() { fn test_typescript_procedure_triggers_subscription_updates() { require_pnpm!(); let mut test = Smoketest::builder().autopublish(false).build(); + let database_name = format!("views-subscribe-typescript-{}", random_string()); test.publish() - .name("views-subscribe-typescript") + .name(&database_name) .source( ModuleLanguage::TypeScript, "views-subscribe-typescript", @@ -933,8 +934,9 @@ fn test_typescript_count_view_subscription_refreshes() { require_pnpm!(); let mut test = Smoketest::builder().autopublish(false).build(); + let database_name = format!("views-count-typescript-{}", random_string()); test.publish() - .name("views-count-typescript") + .name(&database_name) .source( ModuleLanguage::TypeScript, "views-count-typescript", @@ -1031,8 +1033,9 @@ fn test_disconnect_does_not_break_anonymous_view() { fn test_typescript_query_builder_view_query() { require_pnpm!(); let mut test = Smoketest::builder().autopublish(false).build(); + let database_name = format!("views-query-builder-typescript-{}", random_string()); test.publish() - .name("views-query-builder-typescript") + .name(&database_name) .source( ModuleLanguage::TypeScript, "views-query-builder-typescript", diff --git a/tools/ci/README.md b/tools/ci/README.md index 980f290c572..78147590f46 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -102,6 +102,12 @@ Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. +- `--auth-host `: Use a SpacetimeAuth-issued login for remote-server tests. + +This is required for servers that reject direct server-issued logins for privileged operations. + +Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. + - `--dotnet `: - `args `: Additional arguments to pass to the test runner - `--help`: Print help (see a summary with '-h') diff --git a/tools/ci/src/smoketest.rs b/tools/ci/src/smoketest.rs index 456c13caa32..41bc8619841 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/src/smoketest.rs @@ -1,11 +1,13 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, ensure, Result}; +use anyhow::{bail, ensure, Context, Result}; use clap::{Args, Subcommand}; use duct::cmd; +use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; use std::path::Path; use std::process::{Command, Stdio}; use std::{env, fs}; +use tempfile::TempDir; use crate::util; @@ -26,6 +28,15 @@ pub struct SmoketestsArgs { #[arg(long)] server: Option, + /// Use a SpacetimeAuth-issued login for remote-server tests. + /// + /// This is required for servers that reject direct server-issued logins for privileged operations. + /// + /// Optionally accepts an auth host to pass through to `spacetime login`, + /// for example `--auth-host=https://spacetimedb.com`. + #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] + auth_host: Option, + #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, @@ -55,7 +66,7 @@ pub fn run(args: SmoketestsArgs) -> Result<()> { eprintln!("smoketests/mod.rs is up to date."); Ok(()) } - None => run_smoketest(args.server, args.dotnet, args.args), + None => run_smoketest(args.server, args.dotnet, args.auth_host.as_deref(), args.args), } } @@ -126,13 +137,17 @@ fn build_precompiled_modules() -> Result<()> { /// 16 was found to be optimal - higher values cause OS scheduler overhead. const DEFAULT_PARALLELISM: &str = "16"; -fn run_smoketest(server: Option, dotnet: bool, args: Vec) -> Result<()> { +fn run_smoketest(server: Option, dotnet: bool, auth_host: Option<&str>, args: Vec) -> Result<()> { // 1. Build binaries first (single process, no race) build_binaries()?; // 2. Build pre-compiled modules (this also warms the WASM dependency cache) build_precompiled_modules()?; + let cli_path = ensure_binaries_built(); + let base_config_dir = prepare_base_config(&cli_path, server.as_deref(), auth_host)?; + let base_config_path = base_config_dir.path().join("config.toml"); + // 4. Detect whether to use nextest or cargo test let use_nextest = Command::new("cargo") .args(["nextest", "--version"]) @@ -150,7 +165,7 @@ fn run_smoketest(server: Option, dotnet: bool, args: Vec) -> Res let mut cmd = if use_nextest { eprintln!("Running smoketests with cargo nextest...\n"); let mut cmd = Command::new("cargo"); - set_env(&mut cmd, server, dotnet); + set_env(&mut cmd, server, dotnet, auth_host.is_some(), &base_config_path); cmd.args([ "nextest", "run", @@ -172,7 +187,7 @@ fn run_smoketest(server: Option, dotnet: bool, args: Vec) -> Res } else { eprintln!("Running smoketests with cargo test...\n"); let mut cmd = Command::new("cargo"); - set_env(&mut cmd, server, dotnet); + set_env(&mut cmd, server, dotnet, auth_host.is_some(), &base_config_path); cmd.args(["test", "--release", "-p", "spacetimedb-smoketests"]); cmd }; @@ -187,10 +202,77 @@ fn run_smoketest(server: Option, dotnet: bool, args: Vec) -> Res Ok(()) } -fn set_env(cmd: &mut Command, server: Option, dotnet: bool) { +fn prepare_base_config(cli_path: &Path, server: Option<&str>, auth_host: Option<&str>) -> Result { + if server.is_none() && auth_host.is_some() { + bail!("--auth-host requires --server"); + } + + let temp_dir = tempfile::tempdir()?; + let config_path = temp_dir.path().join("config.toml"); + let config_path_str = config_path.to_str().context("invalid temp config path")?; + + // run an arbitrary command in order to initialize the config file + let status = Command::new(cli_path) + .args(["--config-path", config_path_str, "server", "set-default", "local"]) + .status() + .context("failed to initialize smoketest server config")?; + ensure!(status.success(), "spacetime server set-default failed"); + + if let Some(server) = server { + let status = Command::new(cli_path) + .args([ + "--config-path", + config_path_str, + "server", + "edit", + "local", + "--url", + server, + "--yes", + ]) + .status() + .context("failed to edit smoketest server config")?; + ensure!(status.success(), "spacetime server edit failed"); + } + + if let Some(auth_host) = auth_host { + eprintln!("Logging in with SpacetimeAuth for remote smoketests..."); + let mut login = Command::new(cli_path); + login.args(["--config-path", config_path_str, "login"]); + if !auth_host.is_empty() { + login.args(["--auth-host", auth_host]); + } + let status = login.status().context("failed to run spacetime login")?; + ensure!(status.success(), "spacetime login failed"); + } else if server.is_some() { + let status = Command::new(cli_path) + .args([ + "--config-path", + config_path_str, + "login", + "--server-issued-login", + "local", + ]) + .status() + .context("failed to create server-issued smoketest identity")?; + ensure!(status.success(), "spacetime login --server-issued-login failed"); + } + + ensure!( + config_path.exists(), + "smoketest config setup succeeded but did not create {}", + config_path.display() + ); + + Ok(temp_dir) +} + +fn set_env(cmd: &mut Command, server: Option, dotnet: bool, auth_host: bool, base_config_path: &Path) { if let Some(ref server_url) = server { cmd.env("SPACETIME_REMOTE_SERVER", server_url); } + cmd.env("SPACETIME_SMOKETEST_BASE_CONFIG_PATH", base_config_path); + cmd.env("SPACETIME_USE_AUTH_HOST", if auth_host { "1" } else { "0" }); cmd.env("SMOKETESTS_DOTNET", if dotnet { "1" } else { "0" }); }