diff --git a/fs/fstest/compare.go b/fs/fstest/compare.go index 98a9abc..baaaa8c 100644 --- a/fs/fstest/compare.go +++ b/fs/fstest/compare.go @@ -19,34 +19,22 @@ package fstest import ( "fmt" "os" - - "github.com/containerd/continuity" ) // CheckDirectoryEqual compares two directory paths to make sure that // the content of the directories is the same. func CheckDirectoryEqual(d1, d2 string) error { - c1, err := continuity.NewContext(d1) - if err != nil { - return fmt.Errorf("failed to build context: %w", err) - } - - c2, err := continuity.NewContext(d2) - if err != nil { - return fmt.Errorf("failed to build context: %w", err) - } - - m1, err := continuity.BuildManifest(c1) + r1, err := buildResources(d1) if err != nil { - return fmt.Errorf("failed to build manifest: %w", err) + return fmt.Errorf("failed to walk %s: %w", d1, err) } - m2, err := continuity.BuildManifest(c2) + r2, err := buildResources(d2) if err != nil { - return fmt.Errorf("failed to build manifest: %w", err) + return fmt.Errorf("failed to walk %s: %w", d2, err) } - diff := diffResourceList(m1.Resources, m2.Resources) + diff := diffResourceList(r1, r2) if diff.HasDiff() { return fmt.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) } diff --git a/fs/fstest/compare_test.go b/fs/fstest/compare_test.go new file mode 100644 index 0000000..74f67ca --- /dev/null +++ b/fs/fstest/compare_test.go @@ -0,0 +1,273 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fstest + +import ( + "os" + "runtime" + "testing" +) + +func TestCheckDirectoryEqualBasic(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a := Apply( + CreateDir("/d", 0o755), + CreateFile("/d/f1", []byte("hello"), 0o644), + CreateFile("/f2", []byte("world"), 0o600), + ) + if err := a.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err != nil { + t.Fatalf("identical directories should be equal: %v", err) + } +} + +func TestCheckDirectoryEqualDetectsDifference(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a1 := Apply( + CreateFile("/f", []byte("aaa"), 0o644), + ) + a2 := Apply( + CreateFile("/f", []byte("bbb"), 0o644), + ) + if err := a1.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a2.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err == nil { + t.Fatal("directories with different content should not be equal") + } +} + +func TestCheckDirectoryEqualExtraFile(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a := Apply( + CreateFile("/f1", []byte("hello"), 0o644), + ) + if err := a.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a.Apply(d2); err != nil { + t.Fatal(err) + } + // Extra file in d2 + if err := CreateFile("/f2", []byte("extra"), 0o644).Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err == nil { + t.Fatal("directory with extra file should not be equal") + } +} + +func TestCheckDirectoryEqualMissingFile(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a1 := Apply( + CreateFile("/f1", []byte("hello"), 0o644), + CreateFile("/f2", []byte("world"), 0o644), + ) + a2 := Apply( + CreateFile("/f1", []byte("hello"), 0o644), + ) + if err := a1.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a2.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err == nil { + t.Fatal("directory with missing file should not be equal") + } +} + +func TestCheckDirectoryEqualSymlinks(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a := Apply( + CreateFile("/target", []byte("data"), 0o644), + Symlink("target", "/link"), + ) + if err := a.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err != nil { + t.Fatalf("identical symlink directories should be equal: %v", err) + } +} + +func TestCheckDirectoryEqualSymlinkDifference(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a1 := Apply( + CreateFile("/target1", []byte("data"), 0o644), + CreateFile("/target2", []byte("data"), 0o644), + Symlink("target1", "/link"), + ) + a2 := Apply( + CreateFile("/target1", []byte("data"), 0o644), + CreateFile("/target2", []byte("data"), 0o644), + Symlink("target2", "/link"), + ) + if err := a1.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a2.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err == nil { + t.Fatal("directories with different symlink targets should not be equal") + } +} + +func TestCheckDirectoryEqualHardlinks(t *testing.T) { + d1 := t.TempDir() + d2 := t.TempDir() + + a := Apply( + CreateFile("/f1", []byte("hello"), 0o644), + Link("/f1", "/f2"), + ) + if err := a.Apply(d1); err != nil { + t.Fatal(err) + } + if err := a.Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err != nil { + t.Fatalf("identical hardlink directories should be equal: %v", err) + } +} + +func TestCheckDirectoryEqualPermissionDifference(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not support Unix-style file permissions") + } + + d1 := t.TempDir() + d2 := t.TempDir() + + if err := CreateFile("/f", []byte("hello"), 0o644).Apply(d1); err != nil { + t.Fatal(err) + } + if err := CreateFile("/f", []byte("hello"), 0o600).Apply(d2); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqual(d1, d2); err == nil { + t.Fatal("directories with different permissions should not be equal") + } +} + +func TestCheckDirectoryEqualWithApplier(t *testing.T) { + d := t.TempDir() + + a := Apply( + CreateDir("/d", 0o755), + CreateFile("/d/f", []byte("content"), 0o644), + ) + if err := a.Apply(d); err != nil { + t.Fatal(err) + } + + if err := CheckDirectoryEqualWithApplier(d, a); err != nil { + t.Fatalf("directory should equal its applier: %v", err) + } +} + +func TestBuildResources(t *testing.T) { + d := t.TempDir() + + a := Apply( + CreateDir("/a", 0o755), + CreateFile("/a/f1", []byte("one"), 0o644), + CreateFile("/b", []byte("two"), 0o600), + Symlink("b", "/c"), + ) + if err := a.Apply(d); err != nil { + t.Fatal(err) + } + + resources, err := buildResources(d) + if err != nil { + t.Fatal(err) + } + + // Should have 4 entries: /a, /a/f1, /b, /c + if len(resources) != 4 { + t.Fatalf("expected 4 resources, got %d", len(resources)) + } + + // Verify sorted order + for i := 1; i < len(resources); i++ { + if resources[i].path <= resources[i-1].path { + t.Fatalf("resources not sorted: %q <= %q", resources[i].path, resources[i-1].path) + } + } + + // Verify types + for _, r := range resources { + switch r.path { + case "/a": + if !r.mode.IsDir() { + t.Errorf("/a should be directory, got %v", r.mode) + } + case "/a/f1": + if !r.mode.IsRegular() { + t.Errorf("/a/f1 should be regular file, got %v", r.mode) + } + if r.size != 3 { + t.Errorf("/a/f1 should have size 3, got %d", r.size) + } + case "/b": + if !r.mode.IsRegular() { + t.Errorf("/b should be regular file, got %v", r.mode) + } + case "/c": + if r.mode&os.ModeSymlink == 0 { + t.Errorf("/c should be symlink, got %v", r.mode) + } + if r.target != "b" { + t.Errorf("/c target should be 'b', got %q", r.target) + } + } + } +} diff --git a/fs/fstest/continuity_util.go b/fs/fstest/continuity_util.go index 45dd574..9e2e46f 100644 --- a/fs/fstest/continuity_util.go +++ b/fs/fstest/continuity_util.go @@ -19,25 +19,24 @@ package fstest import ( "bytes" "fmt" - - "github.com/containerd/continuity" + "os" ) type resourceUpdate struct { - Original continuity.Resource - Updated continuity.Resource + Original resource + Updated resource } func (u resourceUpdate) String() string { return fmt.Sprintf("%s(mode: %o, uid: %d, gid: %d) -> %s(mode: %o, uid: %d, gid: %d)", - u.Original.Path(), u.Original.Mode(), u.Original.UID(), u.Original.GID(), - u.Updated.Path(), u.Updated.Mode(), u.Updated.UID(), u.Updated.GID(), + u.Original.path, u.Original.mode, u.Original.uid, u.Original.gid, + u.Updated.path, u.Updated.mode, u.Updated.uid, u.Updated.gid, ) } type resourceListDifference struct { - Additions []continuity.Resource - Deletions []continuity.Resource + Additions []resource + Deletions []resource Updates []resourceUpdate } @@ -47,7 +46,7 @@ func (l resourceListDifference) HasDiff() bool { } for _, add := range l.Additions { - if ok := metadataFiles[add.Path()]; !ok { + if ok := metadataFiles[add.path]; !ok { return true } } @@ -58,10 +57,10 @@ func (l resourceListDifference) HasDiff() bool { func (l resourceListDifference) String() string { buf := bytes.NewBuffer(nil) for _, add := range l.Additions { - fmt.Fprintf(buf, "+ %s\n", add.Path()) + fmt.Fprintf(buf, "+ %s\n", add.path) } for _, del := range l.Deletions { - fmt.Fprintf(buf, "- %s\n", del.Path()) + fmt.Fprintf(buf, "- %s\n", del.path) } for _, upt := range l.Updates { fmt.Fprintf(buf, "~ %s\n", upt.String()) @@ -69,17 +68,17 @@ func (l resourceListDifference) String() string { return buf.String() } -// diffManifest compares two resource lists and returns the list +// diffResourceList compares two resource lists and returns the list // of adds updates and deletes, resource lists are not reordered // before doing difference. -func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference { +func diffResourceList(r1, r2 []resource) resourceListDifference { i1 := 0 i2 := 0 var d resourceListDifference for i1 < len(r1) && i2 < len(r2) { - p1 := r1[i1].Path() - p2 := r2[i2].Path() + p1 := r1[i1].path + p2 := r2[i2].path switch { case p1 < p2: d.Deletions = append(d.Deletions, r1[i1]) @@ -112,103 +111,51 @@ func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference { return d } -func compareResource(r1, r2 continuity.Resource) bool { - if r1.Path() != r2.Path() { +func compareResource(r1, r2 resource) bool { + if r1.path != r2.path { return false } - if r1.Mode() != r2.Mode() { + if r1.mode != r2.mode { return false } - if r1.UID() != r2.UID() { + if r1.uid != r2.uid { return false } - if r1.GID() != r2.GID() { + if r1.gid != r2.gid { return false } - // TODO(dmcgowan): Check if is XAttrer - - return compareResourceTypes(r1, r2) + return compareResourceType(r1, r2) } -func compareResourceTypes(r1, r2 continuity.Resource) bool { - switch t1 := r1.(type) { - case continuity.RegularFile: - t2, ok := r2.(continuity.RegularFile) - if !ok { +func compareResourceType(r1, r2 resource) bool { + mode := r1.mode + switch { + case mode.IsRegular(): + if r1.size != r2.size { return false } - return compareRegularFile(t1, t2) - case continuity.Directory: - t2, ok := r2.(continuity.Directory) - if !ok { + if r1.sha256 != r2.sha256 { return false } - return compareDirectory(t1, t2) - case continuity.SymLink: - t2, ok := r2.(continuity.SymLink) - if !ok { + if len(r1.paths) != len(r2.paths) { return false } - return compareSymLink(t1, t2) - case continuity.NamedPipe: - t2, ok := r2.(continuity.NamedPipe) - if !ok { - return false - } - return compareNamedPipe(t1, t2) - case continuity.Device: - t2, ok := r2.(continuity.Device) - if !ok { - return false + for i := range r1.paths { + if r1.paths[i] != r2.paths[i] { + return false + } } - return compareDevice(t1, t2) + return true + case mode.IsDir(): + return true + case mode&os.ModeSymlink != 0: + return r1.target == r2.target + case mode&os.ModeNamedPipe != 0: + return true + case mode&os.ModeDevice != 0: + return r1.major == r2.major && r1.minor == r2.minor default: - // TODO(dmcgowan): Should this panic? - return r1 == r2 - } -} - -func compareRegularFile(r1, r2 continuity.RegularFile) bool { - if r1.Size() != r2.Size() { - return false - } - p1 := r1.Paths() - p2 := r2.Paths() - if len(p1) != len(p2) { - return false - } - for i := range p1 { - if p1[i] != p2[i] { - return false - } - } - d1 := r1.Digests() - d2 := r2.Digests() - if len(d1) != len(d2) { - return false - } - for i := range d1 { - if d1[i] != d2[i] { - return false - } + return true } - - return true -} - -func compareSymLink(r1, r2 continuity.SymLink) bool { - return r1.Target() == r2.Target() -} - -func compareDirectory(r1, r2 continuity.Directory) bool { - return true -} - -func compareNamedPipe(r1, r2 continuity.NamedPipe) bool { - return true -} - -func compareDevice(r1, r2 continuity.Device) bool { - return r1.Major() == r2.Major() && r1.Minor() == r2.Minor() } diff --git a/fs/fstest/mkfs_linux.go b/fs/fstest/mkfs_linux.go index 9510ef1..d7f8676 100644 --- a/fs/fstest/mkfs_linux.go +++ b/fs/fstest/mkfs_linux.go @@ -24,8 +24,10 @@ import ( "github.com/containerd/continuity/testutil/loopback" ) +// WithMkfs creates a loopback device, formats it with the given mkfs command, +// mounts it, and runs f with TMPDIR set to the mount point. +// The caller should ensure root access before calling this function. func WithMkfs(t *testing.T, f func(), mkfs ...string) { - testutil.RequiresRoot(t) mnt := t.TempDir() loop, err := loopback.New(100 << 20) // 100 MB if err != nil { diff --git a/fs/fstest/mkfs_others.go b/fs/fstest/mkfs_others.go index 5e0bef6..4281dc9 100644 --- a/fs/fstest/mkfs_others.go +++ b/fs/fstest/mkfs_others.go @@ -18,7 +18,9 @@ package fstest -import "testing" +import ( + "testing" +) func WithMkfs(t *testing.T, f func(), mkfs ...string) { t.Fatal("WithMkfs requires Linux") diff --git a/fs/fstest/walker.go b/fs/fstest/walker.go new file mode 100644 index 0000000..24070ca --- /dev/null +++ b/fs/fstest/walker.go @@ -0,0 +1,149 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fstest + +import ( + "crypto/sha256" + "io" + "os" + "path/filepath" + "sort" +) + +// resource represents a filesystem entry for directory comparison. +type resource struct { + path string + paths []string // all paths, including hardlinks (sorted) + mode os.FileMode + uid int64 + gid int64 + size int64 + sha256 [sha256.Size]byte // regular files only + target string // symlinks only + major uint64 // devices only + minor uint64 // devices only +} + +// buildResources walks root and returns a sorted list of resources. +func buildResources(root string) ([]resource, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + + type entry struct { + res resource + fi os.FileInfo + } + + // hlKey -> index into entries for the first file with that inode. + hardlinks := map[hardlinkKey]int{} + var entries []entry + + err = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + // Use absolute-style paths like continuity does (rooted at "/"). + rel = "/" + filepath.ToSlash(rel) + if rel == "/." { + // skip root directory itself + return nil + } + + r := resource{ + path: rel, + mode: fi.Mode(), + } + statResource(fi, &r) + + if fi.Mode().IsRegular() { + r.size = fi.Size() + h, err := hashFile(p) + if err != nil { + return err + } + r.sha256 = h + + // Check for hardlink. + if key, ok := getHardlinkKey(fi); ok { + if idx, exists := hardlinks[key]; exists { + // Merge into existing entry. + entries[idx].res.paths = append(entries[idx].res.paths, rel) + return nil + } + hardlinks[key] = len(entries) + } + } else if fi.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(p) + if err != nil { + return err + } + r.target = target + } else if fi.Mode()&os.ModeDevice != 0 { + r.major, r.minor = getDeviceInfo(fi) + } else if fi.Mode()&os.ModeNamedPipe != 0 { + // Check for hardlink on named pipes. + if key, ok := getHardlinkKey(fi); ok { + if idx, exists := hardlinks[key]; exists { + entries[idx].res.paths = append(entries[idx].res.paths, rel) + return nil + } + hardlinks[key] = len(entries) + } + } + + r.paths = []string{rel} + entries = append(entries, entry{res: r, fi: fi}) + return nil + }) + if err != nil { + return nil, err + } + + resources := make([]resource, len(entries)) + for i, e := range entries { + sort.Strings(e.res.paths) + e.res.path = e.res.paths[0] + resources[i] = e.res + } + sort.Slice(resources, func(i, j int) bool { + return resources[i].path < resources[j].path + }) + + return resources, nil +} + +func hashFile(path string) ([sha256.Size]byte, error) { + f, err := os.Open(path) + if err != nil { + return [sha256.Size]byte{}, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return [sha256.Size]byte{}, err + } + var sum [sha256.Size]byte + copy(sum[:], h.Sum(nil)) + return sum, nil +} diff --git a/fs/fstest/walker_unix.go b/fs/fstest/walker_unix.go new file mode 100644 index 0000000..4fab719 --- /dev/null +++ b/fs/fstest/walker_unix.go @@ -0,0 +1,57 @@ +//go:build !windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fstest + +import ( + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +type hardlinkKey struct { + dev uint64 + inode uint64 +} + +func statResource(fi os.FileInfo, r *resource) { + if sys, ok := fi.Sys().(*syscall.Stat_t); ok { + r.uid = int64(sys.Uid) + r.gid = int64(sys.Gid) + } +} + +func getHardlinkKey(fi os.FileInfo) (hardlinkKey, bool) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok || sys.Nlink < 2 { + return hardlinkKey{}, false + } + //nolint:unconvert + return hardlinkKey{dev: uint64(sys.Dev), inode: uint64(sys.Ino)}, true +} + +func getDeviceInfo(fi os.FileInfo) (major, minor uint64) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + //nolint:unconvert + dev := uint64(sys.Rdev) + return uint64(unix.Major(dev)), uint64(unix.Minor(dev)) +} diff --git a/fs/fstest/walker_windows.go b/fs/fstest/walker_windows.go new file mode 100644 index 0000000..f300fd9 --- /dev/null +++ b/fs/fstest/walker_windows.go @@ -0,0 +1,33 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fstest + +import "os" + +type hardlinkKey struct{} + +func statResource(fi os.FileInfo, r *resource) { + // Windows does not support uid/gid. +} + +func getHardlinkKey(fi os.FileInfo) (hardlinkKey, bool) { + return hardlinkKey{}, false +} + +func getDeviceInfo(fi os.FileInfo) (major, minor uint64) { + return 0, 0 +}