From 7992660dcb7d332fd9ec77afa8ce88e16e52e7bb Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Fri, 8 May 2026 17:34:04 +0000 Subject: [PATCH] fork: hardlink snapshot mem-file into snapshot forks Snapshot forks copy the source guest dir into the fork instance dir; the dominant cost is the multi-GB mem-file. Hardlink it instead and skip the file from the directory walk via CopyOptions.SkipRelPaths (introduced for template forks). This is safe because: - snapshot mem-files are immutable - the hypervisor mmaps them MAP_PRIVATE on restore, so fork writes never reach the underlying file - hardlinks survive snapshot deletion via inode refcount, so a deleted snapshot never strands a running fork Falls back to the regular copy walk when no raw mem-file is present. --- lib/instances/snapshot.go | 51 +++++++++++++++++++++++++++++-- lib/instances/snapshot_test.go | 56 ++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 5c8030e9..9d01c451 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "time" "github.com/kernel/hypeman/lib/forkvm" @@ -409,16 +410,27 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if target != nil && target.State == compressionJobStateRunning { m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, target.Target) } - if err := m.ensureSnapshotMemoryReady(ctx, m.paths.SnapshotGuestDir(snapshotID), "", rec.StoredMetadata.HypervisorType); err != nil { + srcDir := m.paths.SnapshotGuestDir(snapshotID) + if err := m.ensureSnapshotMemoryReady(ctx, srcDir, "", rec.StoredMetadata.HypervisorType); err != nil { return nil, fmt.Errorf("prepare snapshot memory for fork: %w", err) } - if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { + copyOpts := forkvm.CopyOptions{} + srcMemPath, srcMemRel, hasSharedMem := snapshotMemHardlinkSource(srcDir) + if hasSharedMem { + copyOpts.SkipRelPaths = []string{srcMemRel} + } + if err := forkvm.CopyGuestDirectoryWithOptions(srcDir, dstDir, copyOpts); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork from snapshot requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) } return nil, fmt.Errorf("clone snapshot payload: %w", err) } + if hasSharedMem { + if err := installForkSnapshotMemHardlink(srcMemPath, dstDir, srcMemRel); err != nil { + return nil, fmt.Errorf("hardlink snapshot mem-file into fork: %w", err) + } + } starter, err := m.getVMStarter(targetHypervisor) if err != nil { @@ -638,3 +650,38 @@ func (m *manager) listSnapshotRecords() ([]snapshotRecord, error) { } return records, nil } + +// snapshotMemHardlinkSource resolves the raw mem-file under a snapshot guest +// dir. Returns its absolute path and forward-slash-relative path for use as a +// CopyOptions skip key. Returns ok=false if no raw mem-file is present (e.g. +// only-compressed snapshot whose decompression failed, or a snapshot kind that +// doesn't carry guest memory). Callers fall back to the regular copy walk. +func snapshotMemHardlinkSource(srcDir string) (absPath, relSlash string, ok bool) { + abs, found := findRawSnapshotMemoryFile(srcDir) + if !found { + return "", "", false + } + rel, err := filepath.Rel(srcDir, abs) + if err != nil { + return "", "", false + } + return abs, filepath.ToSlash(rel), true +} + +// installForkSnapshotMemHardlink hardlinks the source snapshot mem-file into +// the fork's data dir at the matching relative path. Snapshot mem-files are +// immutable and the hypervisor mmaps them MAP_PRIVATE on restore, so all +// forks of a snapshot can safely share the same inode — fork writes never +// reach the underlying file. Hardlinks are FS-local and survive snapshot +// deletion via inode refcount, so a deleted snapshot never strands a fork. +func installForkSnapshotMemHardlink(srcMemPath, dstDir, relSlash string) error { + dstMem := filepath.Join(dstDir, filepath.FromSlash(relSlash)) + if err := os.MkdirAll(filepath.Dir(dstMem), 0o755); err != nil { + return fmt.Errorf("ensure fork mem-file parent dir: %w", err) + } + _ = os.Remove(dstMem) + if err := os.Link(srcMemPath, dstMem); err != nil { + return fmt.Errorf("link snapshot mem-file: %w", err) + } + return nil +} diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index cc634f55..e2238a02 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -280,6 +280,62 @@ func TestForkSnapshotFromCompressedSourceCopiesRawMemory(t *testing.T) { assert.False(t, ok, "forked snapshot payload should not retain compressed memory artifacts from the source snapshot") } +func TestForkSnapshotHardlinksRawMemoryFile(t *testing.T) { + t.Parallel() + + mgr, _ := setupTestManager(t) + ctx := context.Background() + + hvType := mgr.defaultHypervisor + sourceID := "snapshot-fork-hardlink-src" + createStandbySnapshotSourceFixture(t, mgr, sourceID, "snapshot-fork-hardlink-src", hvType) + + snap, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStandby, + Name: "standby-for-fork-hardlink", + }) + require.NoError(t, err) + + // Plant the raw mem-file at the top of the snapshot guest dir so it + // survives applyForkTargetState's snapshot-latest wipe and we can stat + // the fork's hardlinked copy after the call returns. Production layout + // nests it under snapshots/snapshot-latest/memory; the helpers under + // test treat both paths identically via findRawSnapshotMemoryFile. + memContents := []byte("guest memory bytes for hardlink test") + snapshotDir := mgr.paths.SnapshotGuestDir(snap.Id) + snapshotMem := filepath.Join(snapshotDir, "memory-ranges") + require.NoError(t, os.WriteFile(snapshotMem, memContents, 0o644)) + snapshotConfigPath := filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "config.json") + require.NoError(t, os.MkdirAll(filepath.Dir(snapshotConfigPath), 0o755)) + require.NoError(t, os.WriteFile(snapshotConfigPath, []byte(`{}`), 0o644)) + + srcInfo, err := os.Stat(snapshotMem) + require.NoError(t, err) + + forked, err := mgr.ForkSnapshot(ctx, snap.Id, ForkSnapshotRequest{ + Name: "snapshot-fork-hardlink", + TargetState: StateStopped, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), forked.Id) }) + + forkMem := filepath.Join(mgr.paths.InstanceDir(forked.Id), "memory-ranges") + forkInfo, err := os.Stat(forkMem) + require.NoError(t, err, "fork should have a hardlinked memory file alongside its instance dir") + assert.True(t, os.SameFile(srcInfo, forkInfo), "fork mem-file should share an inode with the snapshot mem-file (hardlink)") + + got, err := os.ReadFile(forkMem) + require.NoError(t, err) + assert.Equal(t, memContents, got, "fork mem-file should expose the same bytes as the snapshot mem-file") + + // Hardlinks survive deletion of the source path: removing the snapshot + // drops one reference but leaves the inode alive for the fork. + require.NoError(t, mgr.DeleteSnapshot(ctx, snap.Id)) + stillThere, err := os.ReadFile(forkMem) + require.NoError(t, err, "fork mem-file should remain readable after snapshot deletion") + assert.Equal(t, memContents, stillThere) +} + func createStoppedSnapshotSourceFixture(t *testing.T, mgr *manager, id, name string, hvType hypervisor.Type) { t.Helper() require.NoError(t, mgr.ensureDirectories(id))