Fix running smoketests against a remote server requiring spacetime auth

crates/smoketests/DEVELOP.md

@@ -18,6 +18,26 @@ cargo smoketest test_sql_format
 cargo smoketest "cli::"  # Run all CLI tests
 ```
 
+### Remote Servers
+
+Run against a standalone-compatible remote server with:
+
+```bash
+cargo ci smoketests --server https://example.spacetimedb.com
+```
+
+Maincloud and maincloud staging require SpacetimeAuth-issued tokens rather than
+server-issued tokens. Use `--auth-host` for those:
+
+```bash
+cargo ci smoketests --server https://maincloud.staging.spacetimedb.com --auth-host
+```
+
+The runner invokes `spacetime login` once, then copies that logged-in config
+into each isolated smoketest config. Tests that need throwaway server-issued
+identities should call `require_server_issued_login!()` so they skip in
+SpacetimeAuth mode.
+
 ### WARNING: Stale Binary Risk
 
 **Smoketests use pre-built binaries and DO NOT automatically rebuild them.**

crates/smoketests/src/lib.rs

@@ -78,6 +78,11 @@ pub fn is_remote_server() -> bool {
     remote_server_url().is_some()
 }
 
+/// Returns true if remote smoketests are using a SpacetimeAuth-issued token.
+pub fn is_using_auth_host() -> bool {
+    std::env::var("SPACETIME_USE_AUTH_HOST").ok().as_deref() == Some("1")
+}
+
 /// Skip this test if running against a remote server.
 ///
 /// Use this macro at the start of tests that require a local server,
@@ -107,6 +112,19 @@ macro_rules! require_local_server {
     };
 }
 
+#[macro_export]
+macro_rules! require_server_issued_login {
+    () => {
+        if $crate::is_using_auth_host() {
+            #[allow(clippy::disallowed_macros)]
+            {
+                eprintln!("Skipping test: requires server-issued throwaway identities");
+            }
+            return;
+        }
+    };
+}
+
 #[macro_export]
 macro_rules! require_dotnet {
     () => {
@@ -501,7 +519,7 @@ pub struct PublishBuilder<'a> {
     break_clients: bool,
     num_replicas: Option<u32>,
     organization: Option<String>,
-    force: bool,
+    force: Option<&'static str>,
     stdin_input: Option<String>,
     source: Option<ModuleSource>,
 }
@@ -528,7 +546,7 @@ impl<'a> PublishBuilder<'a> {
             break_clients: false,
             num_replicas: None,
             organization: None,
-            force: true,
+            force: Some("all"),
             stdin_input: None,
             source: None,
         }
@@ -559,13 +577,13 @@ impl<'a> PublishBuilder<'a> {
         self
     }
 
-    pub fn force(mut self, force: bool) -> Self {
+    pub fn force(mut self, force: Option<&'static str>) -> Self {
         self.force = force;
         self
     }
 
     pub fn stdin(mut self, stdin_input: impl Into<String>) -> Self {
-        self.force = false;
+        self.force = None;
         self.stdin_input = Some(stdin_input.into());
         self
     }
@@ -912,6 +930,10 @@ impl SmoketestBuilder {
         let module_name = format!("smoketest_module_{}", random_string());
 
         let config_path = project_dir.path().join("config.toml");
+        if let Ok(base_config_path) = std::env::var("SPACETIME_SMOKETEST_BASE_CONFIG_PATH") {
+            fs::copy(&base_config_path, &config_path)
+                .unwrap_or_else(|err| panic!("failed to copy base smoketest config from {base_config_path}: {err:#}"));
+        }
         let mut smoketest = Smoketest {
             guard,
             _data_dir_fixture: data_dir_fixture,
@@ -1416,7 +1438,7 @@ log = "0.4"
         break_clients: bool,
         num_replicas: Option<u32>,
         organization: Option<&str>,
-        force: bool,
+        force: Option<&str>,
         stdin_input: Option<&str>,
     ) -> Result<String> {
         let start = Instant::now();
@@ -1459,9 +1481,10 @@ log = "0.4"
         // Now publish with --bin-path to skip rebuild
         let publish_start = Instant::now();
         let mut args = vec!["publish", "--server", &self.server_url, "--bin-path", &wasm_path_str];
-
-        if force {
-            args.push("--yes");
+        let force_arg;
+        if let Some(force) = force {
+            force_arg = format!("--yes={force}");
+            args.push(&force_arg);
         }
 
         if clear {

crates/smoketests/tests/smoketests/cli/publish.rs

@@ -16,12 +16,30 @@ fn cli_can_publish_spacetimedb_on_disk() {
 
     let dir = dir.to_string();
     let _ = test
-        .spacetime(&["publish", "--module-path", &dir, "--server", &test.server_url, "foobar"])
+        .spacetime(&[
+            "publish",
+            "--module-path",
+            &dir,
+            "--server",
+            &test.server_url,
+            // Needed when running smoketests against a remote server.
+            "--yes=remote",
+            "foobar",
+        ])
         .unwrap();
 
     // Can republish without error to the same name
     let _ = test
-        .spacetime(&["publish", "--module-path", &dir, "--server", &test.server_url, "foobar"])
+        .spacetime(&[
+            "publish",
+            "--module-path",
+            &dir,
+            "--server",
+            &test.server_url,
+            // Needed when running smoketests against a remote server.
+            "--yes=remote",
+            "foobar",
+        ])
         .unwrap();
 }
 
@@ -232,6 +250,8 @@ fn cli_publish_with_config_but_no_match_uses_cli_args() {
         "publish",
         "--server",
         &test.server_url,
+        // Needed when running smoketests against a remote server.
+        "--yes=remote",
         "cli-db-name",
         "--module-path",
         module_dir.to_str().unwrap(),

crates/smoketests/tests/smoketests/http_routes.rs

@@ -1,6 +1,6 @@
 use regex::Regex;
 use spacetimedb_smoketests::{
-    require_dotnet, require_emscripten, require_pnpm, workspace_root, ModuleLanguage, Smoketest,
+    random_string, require_dotnet, require_emscripten, require_pnpm, workspace_root, ModuleLanguage, Smoketest,
 };
 use std::{fs, path::Path};
 
@@ -1109,9 +1109,10 @@ fn cpp_http_test(name: &str, module_code: &str) -> (Smoketest, String) {
 fn typescript_http_test(name: &str, module_code: &str) -> (Smoketest, String) {
     require_pnpm!();
     let mut test = Smoketest::builder().autopublish(false).build();
+    let database_name = format!("{name}-{}", random_string());
     let identity = test
         .publish()
-        .name(name)
+        .name(&database_name)
         .source(ModuleLanguage::TypeScript, name, module_code)
         .run()
         .unwrap();

crates/smoketests/tests/smoketests/new_user_flow.rs

@@ -1,9 +1,13 @@
-use spacetimedb_smoketests::Smoketest;
+use spacetimedb_smoketests::{require_server_issued_login, Smoketest};
 
 // TODO: This test originally was testing to make sure that our tutorial isn't broken. Since our onboarding has changed we should probably update this test in the future.
 /// Test the entirety of the new user flow.
 #[test]
 fn test_new_user_flow() {
+    // This flow creates a throwaway server-issued identity with `new_identity`,
+    // which is not available when smoketests use SpacetimeAuth login.
+    require_server_issued_login!();
+
     let mut test = Smoketest::builder()
         .precompiled_module("new-user-flow")
         .autopublish(false)

crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs

@@ -36,14 +36,22 @@ fn upgrade_prompt_on_publish() {
     let deny_err = test
         .publish()
         .name(&db_name)
-        .force(false)
+        // Needed when running smoketests against a remote server.
+        .force(Some("remote"))
         .run()
         .unwrap_err()
         .to_string();
     assert!(deny_err.contains("major version upgrade from 1.0 to 2.0"));
     assert!(deny_err.contains("Please type 'upgrade' to accept this change:"));
 
-    let accepted_identity = test.publish().name(&db_name).stdin("upgrade\n").run().unwrap();
+    let accepted_identity = test
+        .publish()
+        .name(&db_name)
+        .stdin("upgrade\n")
+        // Needed when running smoketests against a remote server.
+        .force(Some("remote"))
+        .run()
+        .unwrap();
     assert_eq!(accepted_identity, initial_identity);
 }
 

crates/smoketests/tests/smoketests/quickstart.rs

@@ -766,7 +766,11 @@ log = "0.4"
 
         // Replace server address
         let host = self.test.server_host();
-        let protocol = "http"; // The smoketest server uses http
+        let protocol = if self.test.server_url.starts_with("https://") {
+            "https"
+        } else {
+            "http"
+        };
         main_code = main_code.replace("http://localhost:3000", &format!("{}://{}", protocol, host));
 
         // Write the client code

crates/smoketests/tests/smoketests/servers.rs

@@ -1,9 +1,13 @@
 use regex::Regex;
-use spacetimedb_smoketests::Smoketest;
+use spacetimedb_smoketests::{require_local_server, Smoketest};
 
 /// Verify that we can add and list server configurations
 #[test]
 fn test_servers() {
+    // This only covers local CLI config behavior, so it is not valuable to run
+    // against remote servers.
+    require_local_server!();
+
     let test = Smoketest::builder().autopublish(false).build();
 
     // Add a test server (local-only command, no --server flag needed)

crates/smoketests/tests/smoketests/timestamp_route.rs

@@ -1,10 +1,14 @@
-use spacetimedb_smoketests::{random_string, Smoketest};
+use spacetimedb_smoketests::{random_string, require_server_issued_login, Smoketest};
 
 const TIMESTAMP_TAG: &str = "__timestamp_micros_since_unix_epoch__";
 
 /// Test the /v1/database/{name}/unstable/timestamp endpoint
 #[test]
 fn test_timestamp_route() {
+    // This test creates a throwaway server-issued identity before publishing,
+    // which is not available when smoketests use SpacetimeAuth login.
+    require_server_issued_login!();
+
     let mut test = Smoketest::builder().autopublish(false).build();
 
     let name = random_string();

crates/smoketests/tests/smoketests/views.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use serde_json::{json, Value};
 use spacetimedb_smoketests::{
-    require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest,
+    random_string, require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest,
 };
 
 const STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY: &str =
@@ -881,8 +881,9 @@ fn test_procedure_triggers_subscription_updates() {
 fn test_typescript_procedure_triggers_subscription_updates() {
     require_pnpm!();
     let mut test = Smoketest::builder().autopublish(false).build();
+    let database_name = format!("views-subscribe-typescript-{}", random_string());
     test.publish()
-        .name("views-subscribe-typescript")
+        .name(&database_name)
         .source(
             ModuleLanguage::TypeScript,
             "views-subscribe-typescript",
@@ -933,8 +934,9 @@ fn test_typescript_count_view_subscription_refreshes() {
     require_pnpm!();
 
     let mut test = Smoketest::builder().autopublish(false).build();
+    let database_name = format!("views-count-typescript-{}", random_string());
     test.publish()
-        .name("views-count-typescript")
+        .name(&database_name)
         .source(
             ModuleLanguage::TypeScript,
             "views-count-typescript",
@@ -1031,8 +1033,9 @@ fn test_disconnect_does_not_break_anonymous_view() {
 fn test_typescript_query_builder_view_query() {
     require_pnpm!();
     let mut test = Smoketest::builder().autopublish(false).build();
+    let database_name = format!("views-query-builder-typescript-{}", random_string());
     test.publish()
-        .name("views-query-builder-typescript")
+        .name(&database_name)
         .source(
             ModuleLanguage::TypeScript,
             "views-query-builder-typescript",

tools/ci/README.md

@@ -102,6 +102,12 @@ Usage: smoketests [OPTIONS] [ARGS]... [COMMAND]
 
 When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped.
 
+- `--auth-host <AUTH_HOST>`: Use a SpacetimeAuth-issued login for remote-server tests.
+
+This is required for servers that reject direct server-issued logins for privileged operations.
+
+Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`.
+
 - `--dotnet <DOTNET>`:
 - `args <ARGS>`: Additional arguments to pass to the test runner
 - `--help`: Print help (see a summary with '-h')