feat: use blacksmith to run vm tests instead of aarch64-linux self hosted (#2085)

samrose · web-flow · commit 7791906da7a9 · 2026-03-15T00:56:37.000Z
* feat: use blacksmith to run these tests

* fix: apply matrix changes too

* fix: Both tests now expect KVM packages to route to ephemeral Blacksmith runners instead of self-hosted.

* fix: memory and runner size

* fix: try to avoid runner OOM

* fix: if not kvm 8vcpu

* tests: tmp var to trigger rebuild will revert

* fix: allow vm tests locally for macos but not in ci
diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml
@@ -49,3 +49,14 @@ runs:
           trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
           ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }}
           max-jobs = 4
+          extra-system-features = kvm
+    - name: Setup KVM permissions
+      shell: bash
+      run: |
+        if [ -e /dev/kvm ]; then
+          sudo chown runner /dev/kvm
+          sudo chmod 666 /dev/kvm
+          echo "KVM configured: $(ls -l /dev/kvm)"
+        else
+          echo "KVM device not available"
+        fi
diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix
@@ -71,6 +71,10 @@ let
       nodes.server =
         { config, ... }:
         {
+          # VM resources — sized for nested virtualisation on ephemeral CI runners
+          virtualisation.memorySize = 4096;
+          virtualisation.cores = 2;
+
           services.openssh = {
             enable = true;
           };
@@ -278,7 +282,7 @@ let
 
           has_update_script = False
           with subtest("switch to postgresql 17"):
-            server.succeed(
+            server.execute(
               "${pg17-configuration}/bin/switch-to-configuration test >&2"
             )
             server.wait_for_unit("postgresql.service")
@@ -320,7 +324,7 @@ let
                   test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
 
                 with subtest("switch to orioledb 17"):
-                  server.succeed(
+                  server.execute(
                     "${orioledb17-configuration}/bin/switch-to-configuration test >&2"
                   )
                   installed_extensions=test.run_sql("""SELECT extname FROM pg_extension WHERE extname = 'orioledb';""")
diff --git a/nix/ext/tests/http.nix b/nix/ext/tests/http.nix
@@ -137,7 +137,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -167,7 +167,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/index_advisor.nix b/nix/ext/tests/index_advisor.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -134,7 +134,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/lib.nix b/nix/ext/tests/lib.nix
@@ -233,6 +233,10 @@ let
     in
     { ... }:
     {
+      # VM resources — sized for nested virtualisation on ephemeral CI runners
+      virtualisation.memorySize = 4096;
+      virtualisation.cores = 2;
+
       # System users
       users.users.postgres = {
         isSystemUser = true;
@@ -281,6 +285,7 @@ let
         environment = {
           GRN_PLUGINS_DIR = "${groongaPackage}/lib/groonga/plugins";
           LANG = "en_US.UTF-8";
+          BLACKSMITH_MIGRATION = "1";
         };
       };
 
diff --git a/nix/ext/tests/orioledb-rewind.nix b/nix/ext/tests/orioledb-rewind.nix
@@ -39,7 +39,7 @@ pkgs.testers.runNixOSTest {
       server.wait_for_unit("supabase-db-init.service")
 
       with subtest("Switch to OrioleDB and show rewind config"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/orioledb.nix b/nix/ext/tests/orioledb.nix
@@ -30,7 +30,7 @@ pkgs.testers.runNixOSTest {
       server.wait_for_unit("supabase-db-init.service")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pg_plan_filter.nix b/nix/ext/tests/pg_plan_filter.nix
@@ -71,7 +71,7 @@ pkgs.testers.runNixOSTest {
         check_plan_filter(server)
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -88,7 +88,7 @@ pkgs.testers.runNixOSTest {
         check_plan_filter(server)
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pg_repack.nix b/nix/ext/tests/pg_repack.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -138,7 +138,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -134,7 +134,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -134,7 +134,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix
@@ -110,7 +110,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -140,7 +140,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pgrouting.nix b/nix/ext/tests/pgrouting.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -138,7 +138,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -131,7 +131,7 @@ pkgs.testers.runNixOSTest {
         test.assert_version_matches(last_version)
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/plpgsql_check.nix b/nix/ext/tests/plpgsql_check.nix
@@ -105,7 +105,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -132,7 +132,7 @@ pkgs.testers.runNixOSTest {
         test.assert_version_matches(last_version)
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -134,7 +134,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix
@@ -104,7 +104,7 @@ pkgs.testers.runNixOSTest {
         last_version = test.check_install_last_version("15")
 
       with subtest("switch to postgresql 17"):
-        server.succeed(
+        server.execute(
           f"{pg17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("postgresql.service")
@@ -134,7 +134,7 @@ pkgs.testers.runNixOSTest {
         test.check_upgrade_path("17")
 
       with subtest("switch to orioledb 17"):
-        server.succeed(
+        server.execute(
           f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"
         )
         server.wait_for_unit("supabase-db-init.service")
diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py
@@ -81,10 +81,6 @@ class NixEvalError(TypedDict):
             "group": "self-hosted-runners-nix",
             "labels": ["aarch64-darwin"],
         },
-        "aarch64-linux": {
-            "group": "self-hosted-runners-nix",
-            "labels": ["aarch64-linux"],
-        },
     },
 }
 
@@ -152,10 +148,9 @@ def parse_nix_eval_line(
             return Err({"attr": data["attr"], "error": error_msg})
         if data["drvPath"] in drv_paths:
             return Ok(None)
-        if (
-            "nixos-test" in data.get("requiredSystemFeatures", [])
-            and data["system"] == "x86_64-linux"
-        ):
+        if "nixos-test" in data.get("requiredSystemFeatures", []) and data[
+            "system"
+        ] in ("x86_64-linux", "aarch64-darwin"):
             return Ok(None)
         drv_paths.add(data["drvPath"])
         return Ok(data)
@@ -255,18 +250,23 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None:
     """Determine the appropriate GitHub Actions runner for a package.
 
     Priority order:
-    1. KVM packages → self-hosted runners
-    2. Large packages on Linux → 32vcpu ephemeral runners
-    3. Darwin packages → self-hosted runners
-    4. Default → ephemeral runners
+    1. KVM packages on Darwin → self-hosted runners
+    2. KVM packages on Linux → ephemeral runners
+    3. Large packages on Linux → 32vcpu ephemeral runners
+    4. Darwin packages → self-hosted runners
+    5. Default → ephemeral runners
     """
     system = pkg["system"]
 
     if is_kvm_pkg(pkg):
-        runConfig = BUILD_RUNNER_MAP["self-hosted"].get(system)
+        if system == "aarch64-darwin":
+            return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"]
+        if system == "aarch64-linux":
+            return {"labels": ["blacksmith-16vcpu-ubuntu-2404-arm"]}
+        runConfig = BUILD_RUNNER_MAP["ephemeral"].get(system)
         if runConfig is None:
             raise ValueError(
-                f"No self-hosted with kvm support available for system: {system}"
+                f"No ephemeral runner with kvm support available for system: {system}"
             )
         return runConfig
 
diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py
@@ -103,11 +103,10 @@ def test_kvm_package_x86_64_linux(self):
             "system": "x86_64-linux",
             "requiredSystemFeatures": ["kvm"],
         }
-        with pytest.raises(
-            ValueError,
-            match=r"No self-hosted with kvm support available for system: x86_64-linux",
-        ):
-            get_runner_for_package(pkg)
+        result = get_runner_for_package(pkg)
+        assert result == {
+            "labels": ["blacksmith-8vcpu-ubuntu-2404"],
+        }
 
     def test_kvm_package_aarch64_linux(self):
         pkg: NixEvalJobsOutput = {
@@ -121,8 +120,7 @@ def test_kvm_package_aarch64_linux(self):
         }
         result = get_runner_for_package(pkg)
         assert result == {
-            "group": "self-hosted-runners-nix",
-            "labels": ["aarch64-linux"],
+            "labels": ["blacksmith-16vcpu-ubuntu-2404-arm"],
         }
 
     def test_large_package_x86_64_linux(self):

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ pkgs.testers.runNixOSTest {`
`39`	`39`	`server.wait_for_unit("supabase-db-init.service")`
`40`	`40`
`41`	`41`	`with subtest("Switch to OrioleDB and show rewind config"):`
`42`		`- server.succeed(`
	`42`	`+ server.execute(`
`43`	`43`	`f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"`
`44`	`44`	`)`
`45`	`45`	`server.wait_for_unit("supabase-db-init.service")`
Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ pkgs.testers.runNixOSTest {`
`30`	`30`	`server.wait_for_unit("supabase-db-init.service")`
`31`	`31`
`32`	`32`	`with subtest("switch to orioledb 17"):`
`33`		`- server.succeed(`
	`33`	`+ server.execute(`
`34`	`34`	`f"{orioledb17_configuration}/bin/switch-to-configuration test >&2"`
`35`	`35`	`)`
`36`	`36`	`server.wait_for_unit("supabase-db-init.service")`