chore: stabilize testing (#132)

* Stabilize linux test network subnet allocation

* Add subnet lease allocator for tests

* tests: prewarm ci cache and mirror images via local registry

* tests: prewarm hypervisor binaries to avoid extraction races

* vmm: lock and atomically install extracted cloud-hypervisor binaries

* tests: require prewarmed system binaries only on linux

* ci: consecutive stability check 2 of 5

* ci: consecutive stability check 3 of 5

* ci: consecutive stability check 4 of 5

* ci: consecutive stability check 5 of 5

Author: sjmiller609

.github/workflows/test.yml

@@ -65,6 +65,13 @@ jobs:
       - name: Build
         run: make build
 
+      - name: Prewarm test cache
+        env:
+          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
+        run: |
+          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/linux-amd64"
+          go run ./cmd/test-prewarm
+
       - name: Check gofmt
         run: |
           set -euo pipefail
@@ -91,7 +98,11 @@ jobs:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           TLS_TEST_DOMAIN: "test.hypeman-development.com"
           TLS_ALLOWED_DOMAINS: '*.hypeman-development.com'
-        run: make test TEST_TIMEOUT=20m
+          HYPEMAN_TEST_PREWARM_STRICT: "1"
+          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
+        run: |
+          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/linux-amd64"
+          make test TEST_TIMEOUT=20m
 
   test-darwin:
     runs-on: [self-hosted, macos, arm64]
@@ -123,6 +134,13 @@ jobs:
       - name: Build
         run: make build
 
+      - name: Prewarm test cache
+        env:
+          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
+        run: |
+          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/darwin-arm64"
+          go run ./cmd/test-prewarm
+
       - name: Check gofmt
         run: |
           set -euo pipefail
@@ -142,7 +160,11 @@ jobs:
           GO_TEST_TIMEOUT: 600s
           DEFAULT_HYPERVISOR: vz
           JWT_SECRET: ci-test-secret
-        run: make test
+          HYPEMAN_TEST_PREWARM_STRICT: "1"
+          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
+        run: |
+          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/darwin-arm64"
+          make test
       - name: Cleanup
         if: always()
         run: |

Makefile

@@ -269,9 +269,17 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries
 	if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \
 	if [ -n "$(TEST)" ]; then \
 		echo "Running specific test: $(TEST)"; \
-		sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
+		sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" \
+			"HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \
+			"HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \
+			"HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \
+			go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
 	else \
-		sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
+		sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" \
+			"HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \
+			"HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \
+			"HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \
+			go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
 	fi
 
 # macOS tests (no sudo needed, adds e2fsprogs to PATH)

cmd/test-prewarm/hypervisor_binaries_linux.go

@@ -0,0 +1,28 @@
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/kernel/hypeman/lib/hypervisor/firecracker"
+	"github.com/kernel/hypeman/lib/paths"
+	"github.com/kernel/hypeman/lib/vmm"
+)
+
+func ensureHypervisorBinaries(p *paths.Paths) (int, string, error) {
+	chBinaries := 0
+	for _, version := range vmm.SupportedVersions {
+		if _, err := vmm.GetBinaryPath(p, version); err != nil {
+			return 0, "", fmt.Errorf("cloud-hypervisor %s: %w", version, err)
+		}
+		chBinaries++
+	}
+
+	fcPath, err := firecracker.NewStarter().GetBinaryPath(p, "")
+	if err != nil {
+		return 0, "", fmt.Errorf("firecracker: %w", err)
+	}
+
+	return chBinaries, fcPath, nil
+}

cmd/test-prewarm/hypervisor_binaries_other.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package main
+
+import "github.com/kernel/hypeman/lib/paths"
+
+func ensureHypervisorBinaries(_ *paths.Paths) (int, string, error) {
+	return 0, "", nil
+}