ci: add integration tests

.github/.tool-versions

@@ -0,0 +1,2 @@
+golang 1.25.8
+kernel 6.12.44

.github/actions/build-kernel/action.yml

@@ -0,0 +1,120 @@
+name: "Build Kernel"
+description: "Composite action to build Linux kernels"
+inputs:
+  kernel_version:
+    description: 'Kernel version to build'
+    required: true
+    default: '6.12.44'
+  kernel_arch:
+    description: 'Kernel architecture to build'
+    required: true
+    default: 'x86_64'
+  kernel_nproc:
+    description: 'Number of parallel build processes'
+    required: false
+    # Public runners provide 4 cores; default to that to avoid overloading
+    # https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+    default: '4'
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout code
+      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435  # v3.11.1
+
+    - name: Calculate kernel cache key
+      id: cache-key
+      shell: bash
+      run: |
+          # Hash the kernel config and patches to create a unique cache key
+          CONFIG_FILE="kernel/config-${{ inputs.kernel_version }}-${{ inputs.kernel_arch }}"
+
+          if [ ! -f "$CONFIG_FILE" ]; then
+            echo "Error: Kernel config file $CONFIG_FILE not found"
+            exit 1
+          fi
+
+          # Calculate hash of config file and all patches
+          CONFIG_HASH=$(sha256sum "$CONFIG_FILE" | cut -d' ' -f1)
+          PATCHES_HASH=$(find kernel/patches -type f -name "*.patch" -exec sha256sum {} \; | sort | sha256sum | cut -d' ' -f1)
+
+          # Combine version, arch, config hash, and patches hash
+          CACHE_KEY="kernel-${{ inputs.kernel_version }}-${{ inputs.kernel_arch }}-${CONFIG_HASH:0:8}-${PATCHES_HASH:0:8}"
+
+          echo "cache-key=${CACHE_KEY}" >> $GITHUB_OUTPUT
+          echo "config-hash=${CONFIG_HASH:0:8}" >> $GITHUB_OUTPUT
+          echo "patches-hash=${PATCHES_HASH:0:8}" >> $GITHUB_OUTPUT
+
+          echo "Kernel cache key: ${CACHE_KEY}"
+          echo "Config hash: ${CONFIG_HASH:0:8}"
+          echo "Patches hash: ${PATCHES_HASH:0:8}"
+
+    - name: Check cache for existing kernel
+      id: cache-kernel
+      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57  # v4.2.0
+      with:
+        path: _output/nerdbox-kernel-${{ inputs.kernel_arch }}
+        key: ${{ steps.cache-key.outputs.cache-key }}
+        lookup-only: true
+
+    - name: Build kernel
+      if: steps.cache-kernel.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        docker buildx bake kernel \
+          --set kernel.args.KERNEL_VERSION=${{ inputs.kernel_version }} \
+          --set kernel.args.KERNEL_ARCH=${{ inputs.kernel_arch }} \
+          --set kernel.args.KERNEL_NPROC=${{ inputs.kernel_nproc }}
+
+    - name: Verify kernel artifact
+      if: steps.cache-kernel.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        kernel_file="_output/nerdbox-kernel-${{ inputs.kernel_arch }}"
+        if [ ! -f "$kernel_file" ]; then
+          echo "Error: Kernel file $kernel_file not found after build"
+          exit 1
+        fi
+
+        echo "Kernel built successfully:"
+        ls -lh "$kernel_file"
+        file "$kernel_file"
+
+    - name: Save kernel to cache
+      if: steps.cache-kernel.outputs.cache-hit != 'true'
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57  # v4.2.0
+      with:
+        path: _output/nerdbox-kernel-${{ inputs.kernel_arch }}
+        key: ${{ steps.cache-key.outputs.cache-key }}
+
+    - name: Upload kernel artifact
+      if: steps.cache-kernel.outputs.cache-hit != 'true'
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+      with:
+        name: nerdbox-kernel-${{ inputs.kernel_version }}-${{ inputs.kernel_arch }}
+        path: _output/nerdbox-kernel-${{ inputs.kernel_arch }}
+        retention-days: 90
+        if-no-files-found: error
+
+    - name: Cache summary
+      shell: bash
+      run: |
+        echo "## Kernel Build Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "- **Version**: ${{ inputs.kernel_version }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Architecture**: ${{ inputs.kernel_arch }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Cache Key**: \`${{ steps.cache-key.outputs.cache-key }}\`" >> $GITHUB_STEP_SUMMARY
+        echo "- **Config Hash**: ${{ steps.cache-key.outputs.config-hash }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Patches Hash**: ${{ steps.cache-key.outputs.patches-hash }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Cache Hit**: ${{ steps.cache-kernel.outputs.cache-hit == 'true' && '✅ Yes (reused existing)' || '❌ No (built from scratch)' }}" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        if [ -f "_output/nerdbox-kernel-${{ inputs.kernel_arch }}" ]; then
+          echo "### Kernel Details" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          ls -lh "_output/nerdbox-kernel-${{ inputs.kernel_arch }}" >> $GITHUB_STEP_SUMMARY
+          file "_output/nerdbox-kernel-${{ inputs.kernel_arch }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+        fi

.github/workflows/build-kernel.yml

@@ -0,0 +1,40 @@
+name: Build Kernel
+
+on:
+  workflow_dispatch:
+    inputs:
+      kernel_version:
+        description: 'Kernel version to build'
+        required: true
+        default: '6.12.44'
+      kernel_nproc:
+        description: 'Number of parallel build processes'
+        required: false
+        # Public runners provide 4 cores; default to that to avoid overloading
+        # https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+        default: '4'
+
+permissions:
+  contents: read
+
+jobs:
+  build-kernel:
+    name: Build Kernel ${{ inputs.kernel_version }}-${{ matrix.kernel_arch }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            kernel_arch: x86_64
+
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+
+      - uses: ./.github/actions/build-kernel
+        with:
+          kernel_version: ${{ inputs.kernel_version }}
+          kernel_arch: ${{ matrix.kernel_arch }}
+          kernel_nproc: ${{ inputs.kernel_nproc }}

.github/workflows/ci.yml

@@ -10,6 +10,23 @@ permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read
 
 jobs:
+  setup:
+    name: Setup
+    runs-on: ubuntu-latest
+    outputs:
+      kernel-version: ${{ steps.set-vars.outputs.kernel-version }}
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        with:
+          sparse-checkout: |
+            .github/.tool-versions
+
+      - name: Set variables
+        id: set-vars
+        run: |
+          kernel_version=$(grep -E '^kernel [0-9.]+$' .github/.tool-versions | sed -E 's/^kernel ([0-9.]+)$/\1/')
+          echo "kernel-version=${kernel_version}" >> $GITHUB_OUTPUT
+
   linters:
     permissions:
       contents: read  # for actions/checkout to fetch code
@@ -37,7 +54,9 @@ jobs:
           path: src/github.com/containerd/nerdbox
           fetch-depth: 100
 
-      - uses: ./src/github.com/containerd/nerdbox/.github/actions/install-go
+      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
+        with:
+          go-version-file: 'src/github.com/containerd/nerdbox/.github/.tool-versions'
 
       - uses: containerd/project-checks@d7751f3c375b8fe4a84c02a068184ee4c1f59bc4  # v1.2.2
         if: github.repository == 'containerd/nerdbox'
@@ -73,7 +92,9 @@ jobs:
           repository: containerd/containerd
           path: src/github.com/containerd/containerd
 
-      - uses: ./src/github.com/containerd/nerdbox/.github/actions/install-go
+      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
+        with:
+          go-version-file: 'src/github.com/containerd/nerdbox/.github/.tool-versions'
 
       - name: Set env
         shell: bash
@@ -86,3 +107,125 @@ jobs:
       - run: script/install-proto-tools
       - run: make proto-fmt
       - run: make check-protos check-api-descriptors
+
+  #
+  # Build kernels on cache miss
+  #
+  build-kernels:
+    name: Build Kernels (if needed)
+    runs-on: ${{ matrix.os }}
+    needs: setup
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            arch: x86_64
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: ./.github/actions/build-kernel
+        with:
+          kernel_version: ${{ needs.setup.outputs.kernel-version }}
+          kernel_arch: ${{ matrix.arch }}
+
+  #
+  # Integration tests
+  #
+  integration:
+    name: Integration Tests
+    needs: [setup, build-kernels]
+    # Always run after kernel builds complete (whether they were cached or not)
+    if: |
+      always() &&
+      (needs.build-kernels.result == 'success' || needs.build-kernels.result == 'skipped')
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            arch: x86_64
+
+    steps:
+      - name: Enable KVM group perms
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          sudo usermod -aG kvm $USER
+
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+
+      - name: Calculate kernel cache key
+        id: cache-key
+        run: |
+          # Hash the kernel config and patches to create a unique cache key
+          CONFIG_FILE="kernel/config-${{ needs.setup.outputs.kernel-version }}-${{ matrix.arch }}"
+
+          if [ ! -f "$CONFIG_FILE" ]; then
+            echo "Error: Kernel config file $CONFIG_FILE not found"
+            exit 1
+          fi
+
+          # Calculate hash of config file and all patches
+          CONFIG_HASH=$(sha256sum "$CONFIG_FILE" | cut -d' ' -f1)
+          PATCHES_HASH=$(find kernel/patches -type f -name "*.patch" -exec sha256sum {} \; | sort | sha256sum | cut -d' ' -f1)
+
+          # Combine version, arch, config hash, and patches hash
+          CACHE_KEY="kernel-${{ needs.setup.outputs.kernel-version }}-${{ matrix.arch }}-${CONFIG_HASH:0:8}-${PATCHES_HASH:0:8}"
+
+          echo "cache-key=${CACHE_KEY}" >> $GITHUB_OUTPUT
+          echo "Kernel cache key: ${CACHE_KEY}"
+
+      - name: Restore cached kernel
+        id: cache-kernel
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57  # v4.2.0
+        with:
+          path: _output/nerdbox-kernel-${{ matrix.arch }}
+          key: ${{ steps.cache-key.outputs.cache-key }}
+
+      - name: Verify kernel from cache
+        run: |
+          if [ "${{ steps.cache-kernel.outputs.cache-hit }}" = "true" ]; then
+            echo "✅ Kernel restored from cache"
+          else
+            echo "❌ Kernel not in cache - this should not happen after build-kernels-on-demand"
+            exit 1
+          fi
+          ls -lh _output/nerdbox-kernel-${{ matrix.arch }}
+          file _output/nerdbox-kernel-${{ matrix.arch }}
+
+      - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435  # v3.11.1
+
+      - name: Build remaining artifacts (initrd and shim)
+        run: |
+          echo "Building host and guest binaries:"
+          docker buildx bake host-binaries guest-binaries
+
+      - name: Verify all artifacts
+        run: |
+          echo "Verifying build artifacts:"
+          ls -lh _output/
+          echo ""
+          echo "Kernel:"
+          file _output/nerdbox-kernel-${{ matrix.arch }}
+          echo ""
+          echo "Initrd:"
+          file _output/nerdbox-initrd
+          echo ""
+          echo "Shim:"
+          file _output/containerd-shim-nerdbox-v1
+
+      - name: Add _output to PATH
+        run: echo "$(pwd)/_output" >> $GITHUB_PATH
+
+      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
+        with:
+          go-version-file: '.github/.tool-versions'
+
+      - name: Run integration tests
+        run: go test -v ./integration/...

Dockerfile

@@ -18,7 +18,7 @@
 # Build the Linux kernel, initrd ,and containerd shim for running nerbox
 
 ARG XX_VERSION=1.9.0
-ARG GO_VERSION=1.25.5
+ARG GO_VERSION=1.25.8
 ARG BASE_DEBIAN_DISTRO="bookworm"
 ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}"
 ARG GOLANGCI_LINT_VERSION=2.7.2

Makefile

@@ -23,6 +23,7 @@ ONI = "👹"
 
 ARCH = $(shell uname -m)
 OS = $(shell uname -s)
+
 LDFLAGS_x86_64_Linux = -lkrun
 LDFLAGS_aarch64_Linux = -lkrun
 LDFLAGS_arm64_Darwin = -L/opt/homebrew/lib -lkrun
@@ -52,10 +53,13 @@ GO_STATIC_LDFLAGS := -ldflags '-extldflags "-static" $(LDFLAGS) $(EXTRA_LDFLAGS)
 MODULE_NAME=$(shell go list -m)
 API_PACKAGES=$(shell ($(GO) list ${GO_TAGS} ./... | grep /api/ ))
 
-.PHONY: clean all validate lint generate protos check-protos check-api-descriptors proto-fmt shell
+.PHONY: clean all build validate lint generate protos check-protos check-api-descriptors proto-fmt shell
+
+all: build
 
-all:
-	$(BUILDX) bake
+build:
+	@echo "$(WHALE) $@"
+	HOST_OS=$(shell uname -s | tr '[:upper:]' '[:lower:]') KERNEL_ARCH=$(ARCH) $(BUILDX) bake
 
 _output/containerd-shim-nerdbox-v1: cmd/containerd-shim-nerdbox-v1 FORCE
 	@echo "$(WHALE) $@"