diff --git a/AUTHORS b/AUTHORS index aa7b8ffb6a..286ff3e47d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -81,3 +81,4 @@ List of contributors, in chronological order: * Brian Witt (https://github.com/bwitt) * Ales Bregar (https://github.com/abregar) * Tim Foerster (https://github.com/tonobo) +* Philip Cramer (https://github.com/PhilipCramer) diff --git a/api/mirror.go b/api/mirror.go index 60c8f23c7f..f0f98e3086 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -72,6 +72,8 @@ type mirrorCreateParams struct { DownloadUdebs bool ` json:"DownloadUdebs"` // Set "true" to mirror installer files DownloadInstaller bool ` json:"DownloadInstaller"` + // Set "true" to mirror AppStream (DEP-11) metadata + DownloadAppStream bool ` json:"DownloadAppStream"` // Set "true" to include dependencies of matching packages when filtering FilterWithDeps bool ` json:"FilterWithDeps"` // Set "true" to skip if the given components are in the Release file @@ -123,7 +125,7 @@ func apiMirrorsCreate(c *gin.Context) { } repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures, - b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller) + b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream) if err != nil { AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err)) @@ -421,6 +423,14 @@ func apiMirrorsUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + if remote.DownloadAppStream && !remote.IsFlat() { + err = remote.DownloadAppStreamFiles(out, downloader, + context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + if remote.Filter != "" { var filterQuery deb.PackageQuery diff --git a/api/mirror_test.go b/api/mirror_test.go index 8b9b751ffd..cb64d485b7 100644 --- a/api/mirror_test.go +++ b/api/mirror_test.go @@ -26,6 +26,21 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) { c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}") } +func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) { + body, err := json.Marshal(gin.H{ + "Name": "test-flat-appstream", + "ArchiveURL": "http://example.com/repo/", + "Distribution": "./", + "DownloadAppStream": true, + }) + c.Assert(err, IsNil) + + response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body)) + c.Assert(err, IsNil) + c.Check(response.Code, Equals, 400) + c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*") +} + func (s *MirrorSuite) TestCreateMirror(c *C) { c.ExpectFailure("Need to mock downloads") body, err := json.Marshal(gin.H{ diff --git a/cmd/db_cleanup.go b/cmd/db_cleanup.go index c079955e5c..f3b52b2345 100644 --- a/cmd/db_cleanup.go +++ b/cmd/db_cleanup.go @@ -26,6 +26,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error { // collect information about references packages... existingPackageRefs := deb.NewPackageRefList() + referencedAppStreamFiles := []string{} // used only in verbose mode to report package use source packageRefSources := map[string][]string{} @@ -55,6 +56,10 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error { } } + for _, poolPath := range repo.AppStreamFiles { + referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath) + } + return nil }) if err != nil { @@ -118,6 +123,11 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error { return nil }) } + + for _, poolPath := range snapshot.AppStreamFiles { + referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath) + } + return nil }) if err != nil { @@ -236,6 +246,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error { return err } + referencedFiles = append(referencedFiles, referencedAppStreamFiles...) sort.Strings(referencedFiles) context.Progress().ShutdownBar() diff --git a/cmd/mirror_create.go b/cmd/mirror_create.go index 049dd2ebb4..d8a910d793 100644 --- a/cmd/mirror_create.go +++ b/cmd/mirror_create.go @@ -20,6 +20,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources") downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool) downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool) + downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool) ignoreSignatures := context.Config().GpgDisableVerify if context.Flags().IsSet("ignore-signatures") { ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool) @@ -41,7 +42,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { } repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), - downloadSources, downloadUdebs, downloadInstaller) + downloadSources, downloadUdebs, downloadInstaller, downloadAppStream) if err != nil { return fmt.Errorf("unable to create mirror: %s", err) } @@ -100,6 +101,7 @@ Example: } cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") diff --git a/cmd/mirror_edit.go b/cmd/mirror_edit.go index f95d97368b..7c7381143f 100644 --- a/cmd/mirror_edit.go +++ b/cmd/mirror_edit.go @@ -35,6 +35,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error { repo.Filter = flag.Value.String() // allows file/stdin with @ case "filter-with-deps": repo.FilterWithDeps = flag.Value.Get().(bool) + case "with-appstream": + repo.DownloadAppStream = flag.Value.Get().(bool) case "with-installer": repo.DownloadInstaller = flag.Value.Get().(bool) case "with-sources": @@ -53,6 +55,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to edit: flat mirrors don't support udebs") } + if repo.IsFlat() && repo.DownloadAppStream { + return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata") + } + if repo.Filter != "" { _, err = query.Parse(repo.Filter) if err != nil { @@ -107,6 +113,7 @@ Example: AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin") cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") diff --git a/cmd/mirror_show.go b/cmd/mirror_show.go index 00018fb848..b106c65676 100644 --- a/cmd/mirror_show.go +++ b/cmd/mirror_show.go @@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error { downloadUdebs = Yes } fmt.Printf("Download .udebs: %s\n", downloadUdebs) + downloadAppStream := No + if repo.DownloadAppStream { + downloadAppStream = Yes + } + fmt.Printf("Download AppStream: %s\n", downloadAppStream) if repo.Filter != "" { fmt.Printf("Filter: %s\n", repo.Filter) filterWithDeps := No diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index df28efa71b..d67fce3ed6 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -64,6 +64,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to update: %s", err) } + if repo.DownloadAppStream && !repo.IsFlat() { + context.Progress().Printf("Downloading AppStream metadata...\n") + err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(), + context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + } + if repo.Filter != "" { context.Progress().Printf("Applying filter...\n") var filterQuery deb.PackageQuery diff --git a/completion.d/_aptly b/completion.d/_aptly index a1f4d85964..6a70dfe3ce 100644 --- a/completion.d/_aptly +++ b/completion.d/_aptly @@ -185,6 +185,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could $keyring \ "-with-sources=[download source packages in addition to binary packages]:$bool" \ "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \ + "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \ "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components" ;; list) @@ -224,6 +225,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \ "-with-sources=[download source packages in addition to binary packages]:$bool" \ "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \ + "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \ "(-)2:mirror name:$mirrors" ;; search) diff --git a/completion.d/aptly b/completion.d/aptly index aa5b506fa7..bdfbe722c7 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -203,7 +203,7 @@ _aptly() "create") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur})) + COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur})) return 0 fi fi @@ -211,7 +211,7 @@ _aptly() "edit") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur})) + COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) fi diff --git a/deb/package_test.go b/deb/package_test.go index e2574fc854..165716b524 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) { } func (s *PackageSuite) TestNewInstallerFromPara(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false) downloader := http.NewFakeDownloader() downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs") downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list") diff --git a/deb/publish.go b/deb/publish.go index 13d8ca369e..c930d322f7 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1053,6 +1053,38 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } + // Pass-through AppStream (DEP-11) files from snapshots + for component, item := range p.sourceItems { + if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 { + continue + } + + prefix := component + "/" + for relPath, poolPath := range item.snapshot.AppStreamFiles { + if !strings.HasPrefix(relPath, prefix) { + continue + } + withinComponent := strings.TrimPrefix(relPath, prefix) + + poolFile, err := packagePool.Open(poolPath) + if err != nil { + return fmt.Errorf("unable to open AppStream file from pool: %v", err) + } + + bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter() + if err != nil { + poolFile.Close() + return fmt.Errorf("unable to generate AppStream index: %v", err) + } + + _, err = bufio.NewReader(poolFile).WriteTo(bufWriter) + poolFile.Close() + if err != nil { + return fmt.Errorf("unable to write AppStream file: %v", err) + } + } + } + udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) diff --git a/deb/publish_test.go b/deb/publish_test.go index 8037d99483..37c1525ed5 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -13,6 +13,7 @@ import ( "github.com/aptly-dev/aptly/database" "github.com/aptly-dev/aptly/database/goleveldb" "github.com/aptly-dev/aptly/files" + "github.com/aptly-dev/aptly/utils" "github.com/ugorji/go/codec" . "gopkg.in/check.v1" @@ -115,7 +116,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.reflist = NewPackageRefListFromPackageList(s.list) - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) repo.packageRefs = s.reflist _ = s.factory.RemoteRepoCollection().Add(repo) @@ -425,6 +426,81 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { c.Assert(err, IsNil) } +func (s *PublishedRepoSuite) TestPublishAppStream(c *C) { + // Components + icons + content1 := []byte("DEP-11 test content for Components-amd64.yml.gz") + tmpFile1 := filepath.Join(c.MkDir(), "Components-amd64.yml.gz") + c.Assert(os.WriteFile(tmpFile1, content1, 0644), IsNil) + + checksums1 := utils.ChecksumInfo{Size: int64(len(content1))} + poolPath1, err := s.packagePool.Import(tmpFile1, "Components-amd64.yml.gz", &checksums1, false, s.cs) + c.Assert(err, IsNil) + + content2 := []byte("DEP-11 icons tar data") + tmpFile2 := filepath.Join(c.MkDir(), "icons-48x48.tar.gz") + c.Assert(os.WriteFile(tmpFile2, content2, 0644), IsNil) + + checksums2 := utils.ChecksumInfo{Size: int64(len(content2))} + poolPath2, err := s.packagePool.Import(tmpFile2, "icons-48x48.tar.gz", &checksums2, false, s.cs) + c.Assert(err, IsNil) + + // Include contrib file that should be skipped + contribContent := []byte("DEP-11 contrib content") + tmpFile3 := filepath.Join(c.MkDir(), "Components-contrib.yml.gz") + c.Assert(os.WriteFile(tmpFile3, contribContent, 0644), IsNil) + + checksums3 := utils.ChecksumInfo{Size: int64(len(contribContent))} + poolPath3, err := s.packagePool.Import(tmpFile3, "Components-contrib.yml.gz", &checksums3, false, s.cs) + c.Assert(err, IsNil) + + s.snapshot.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": poolPath1, + "main/dep11/icons-48x48.tar.gz": poolPath2, + "contrib/dep11/Components-amd64.yml.gz": poolPath3, + } + + err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") + c.Assert(err, IsNil) + + // Both main files should exist + appstreamPath1 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/Components-amd64.yml.gz") + c.Check(appstreamPath1, PathExists) + actual1, err := os.ReadFile(appstreamPath1) + c.Assert(err, IsNil) + c.Check(actual1, DeepEquals, content1) + + appstreamPath2 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/icons-48x48.tar.gz") + c.Check(appstreamPath2, PathExists) + actual2, err := os.ReadFile(appstreamPath2) + c.Assert(err, IsNil) + c.Check(actual2, DeepEquals, content2) + + // Contrib file should not appear + contribPath := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/contrib/dep11/Components-amd64.yml.gz") + _, statErr := os.Stat(contribPath) + c.Check(os.IsNotExist(statErr), Equals, true) + + // Release file should reference AppStream files + rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release")) + c.Assert(err, IsNil) + defer rf.Close() + + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() + c.Assert(err, IsNil) + + c.Check(st["MD5Sum"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*") + c.Check(st["SHA256"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*") + + // Pool open error + s.snapshot.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": "nonexistent/pool/path", + } + + err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") + c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*") +} + func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) diff --git a/deb/remote.go b/deb/remote.go index 8776b65a4b..a1a79d689b 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -70,6 +70,10 @@ type RemoteRepo struct { DownloadUdebs bool // Should we download installer files? DownloadInstaller bool + // Should we download AppStream (DEP-11) metadata? + DownloadAppStream bool + // AppStream files: relative path (e.g. "main/dep11/Components-amd64.yml.gz") → pool path + AppStreamFiles map[string]string `codec:"AppStreamFiles" json:"-"` // Packages for json output Packages []string `codec:"-" json:",omitempty"` // "Snapshot" of current list of packages @@ -82,7 +86,7 @@ type RemoteRepo struct { // NewRemoteRepo creates new instance of Debian remote repository with specified params func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, - architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) { + architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool, downloadAppStream bool) (*RemoteRepo, error) { result := &RemoteRepo{ UUID: uuid.NewString(), Name: name, @@ -93,6 +97,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone DownloadSources: downloadSources, DownloadUdebs: downloadUdebs, DownloadInstaller: downloadInstaller, + DownloadAppStream: downloadAppStream, } err := result.prepare() @@ -111,6 +116,9 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone if result.DownloadUdebs { return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos") } + if result.DownloadAppStream { + return nil, fmt.Errorf("AppStream (DEP-11) metadata isn't supported for flat repos") + } result.Components = nil } @@ -147,6 +155,9 @@ func (repo *RemoteRepo) String() string { if repo.DownloadInstaller { srcFlag += " [installer]" } + if repo.DownloadAppStream { + srcFlag += " [appstream]" + } distribution := repo.Distribution if distribution == "" { distribution = "./" @@ -264,6 +275,82 @@ func (repo *RemoteRepo) InstallerPath(component string, architecture string) str return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture) } +// AppStreamPaths returns dep11 file paths from ReleaseFiles for a given component +func (repo *RemoteRepo) AppStreamPaths(component string) []string { + prefix := component + "/dep11/" + var paths []string + for path := range repo.ReleaseFiles { + if strings.HasPrefix(path, prefix) { + paths = append(paths, path) + } + } + sort.Strings(paths) + return paths +} + +// DownloadAppStreamFiles downloads AppStream (DEP-11) metadata files and imports them into the pool +func (repo *RemoteRepo) DownloadAppStreamFiles(progress aptly.Progress, d aptly.Downloader, + packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, ignoreChecksums bool) error { + + repo.AppStreamFiles = make(map[string]string) + + for _, component := range repo.Components { + paths := repo.AppStreamPaths(component) + if len(paths) == 0 { + continue + } + + for _, relativePath := range paths { + info, ok := repo.ReleaseFiles[relativePath] + if !ok { + continue + } + + url := repo.IndexesRootURL().ResolveReference(&url.URL{Path: relativePath}).String() + + if progress != nil { + progress.Printf("Downloading AppStream file %s...\n", relativePath) + } + + tempDir, err := os.MkdirTemp("", "aptly-appstream-*") + if err != nil { + return fmt.Errorf("unable to create temp dir for AppStream file %s: %s", relativePath, err) + } + + tempPath := path.Join(tempDir, path.Base(relativePath)) + + var expected *utils.ChecksumInfo + if !ignoreChecksums { + expected = &info + } + + err = d.DownloadWithChecksum(gocontext.TODO(), url, tempPath, expected, ignoreChecksums) + if err != nil { + _ = os.RemoveAll(tempDir) + // Skip files that are not found (some repos list dep11 files but don't serve them) + if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) { + if progress != nil { + progress.ColoredPrintf("@y[!]@| @!skipping AppStream file %s: not found@|", relativePath) + } + continue + } + return fmt.Errorf("unable to download AppStream file %s: %s", relativePath, err) + } + + basename := path.Base(relativePath) + poolPath, err := packagePool.Import(tempPath, basename, &info, true, checksumStorage) + _ = os.RemoveAll(tempDir) + if err != nil { + return fmt.Errorf("unable to import AppStream file %s: %s", relativePath, err) + } + + repo.AppStreamFiles[relativePath] = poolPath + } + } + + return nil +} + // PackageURL returns URL of package file relative to repository root // architecture func (repo *RemoteRepo) PackageURL(filename string) *url.URL { diff --git a/deb/remote_test.go b/deb/remote_test.go index 881685850b..4154e0b8a2 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -2,6 +2,7 @@ package deb import ( "errors" + "fmt" "io" "os" "sort" @@ -90,8 +91,8 @@ type RemoteRepoSuite struct { var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false) - s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false, false) s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.progress = console.NewProgress(false) s.db, _ = goleveldb.NewOpenDB(c.MkDir()) @@ -108,7 +109,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) { } func (s *RemoteRepoSuite) TestInvalidURL(c *C) { - _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false) + _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*") } @@ -117,12 +118,15 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) { c.Check(s.flat.Distribution, Equals, "./") c.Check(s.flat.Components, IsNil) - flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false) + flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false, false) c.Check(flat2.IsFlat(), Equals, true) c.Check(flat2.Distribution, Equals, "./binary/") - _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false) + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false, false) c.Check(err, ErrorMatches, "components aren't supported for flat repos") + + _, err = NewRemoteRepo("fl", "http://some.repo/", "./", []string{}, []string{}, false, false, false, true) + c.Check(err, ErrorMatches, "AppStream \\(DEP-11\\) metadata isn't supported for flat repos") } func (s *RemoteRepoSuite) TestString(c *C) { @@ -135,6 +139,43 @@ func (s *RemoteRepoSuite) TestString(c *C) { s.flat.DownloadSources = true c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]") c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]") + + s.repo.DownloadAppStream = true + c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer] [appstream]") + + // AppStream is not supported for flat repos, so no flat test here +} + +func (s *RemoteRepoSuite) TestAppStreamPaths(c *C) { + s.repo.ReleaseFiles = nil + c.Check(s.repo.AppStreamPaths("main"), DeepEquals, []string(nil)) + + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/binary-amd64/Packages": {Size: 100}, + "main/dep11/Components-amd64.yml.gz": {Size: 200}, + "main/dep11/Components-i386.yml.gz": {Size: 300}, + "main/dep11/icons-48x48.tar.gz": {Size: 400}, + "contrib/dep11/Components-amd64.yml.gz": {Size: 500}, + "main/source/Sources": {Size: 600}, + } + + paths := s.repo.AppStreamPaths("main") + c.Check(paths, DeepEquals, []string{ + "main/dep11/Components-amd64.yml.gz", + "main/dep11/Components-i386.yml.gz", + "main/dep11/icons-48x48.tar.gz", + }) + + paths = s.repo.AppStreamPaths("contrib") + c.Check(paths, DeepEquals, []string{ + "contrib/dep11/Components-amd64.yml.gz", + }) + + paths = s.repo.AppStreamPaths("non-free") + c.Check(paths, DeepEquals, []string(nil)) + + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{} + c.Check(s.repo.AppStreamPaths("main"), DeepEquals, []string(nil)) } func (s *RemoteRepoSuite) TestNumPackages(c *C) { @@ -236,13 +277,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) { } func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false, false) err := s.repo.Fetch(s.downloader, nil, true) c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*") } func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false, false) err := s.repo.Fetch(s.downloader, nil, true) c.Assert(err, ErrorMatches, "component xyz not available in repo.*") } @@ -684,6 +725,74 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(s.flat.packageRefs, NotNil) } +func (s *RemoteRepoSuite) TestDownloadAppStreamFiles(c *C) { + // No dep11 entries + s.repo.Components = []string{"main"} + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/binary-amd64/Packages": {Size: 100}, + "main/source/Sources": {Size: 200}, + } + + err := s.repo.DownloadAppStreamFiles(s.progress, s.downloader, s.packagePool, s.cs, false) + c.Assert(err, IsNil) + c.Check(s.repo.AppStreamFiles, HasLen, 0) + + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/dep11/Components-amd64.yml.gz": {Size: 16}, + "main/dep11/icons-48x48.tar.gz": {Size: 16}, + "main/binary-amd64/Packages": {Size: 100}, + } + + downloader := http.NewFakeDownloader() + downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components") + downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz", "dep11-icons-data") + + err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false) + c.Assert(err, IsNil) + c.Check(s.repo.AppStreamFiles, HasLen, 2) + c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "") + c.Check(s.repo.AppStreamFiles["main/dep11/icons-48x48.tar.gz"], Not(Equals), "") + + // 404 skipped + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/dep11/Components-amd64.yml.gz": {Size: 16}, + "main/dep11/icons-48x48.tar.gz": {Size: 15}, + } + + downloader = http.NewFakeDownloader() + downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components") + downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz", &http.Error{Code: 404, URL: "http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz"}) + + err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false) + c.Assert(err, IsNil) + c.Check(s.repo.AppStreamFiles, HasLen, 1) + c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "") + + // Generic download error propagated + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/dep11/Components-amd64.yml.gz": {Size: 18}, + } + + downloader = http.NewFakeDownloader() + downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", fmt.Errorf("connection refused")) + + err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false) + c.Assert(err, ErrorMatches, "unable to download AppStream file.*connection refused") + + // Bypass checksum validation + s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{ + "main/dep11/Components-amd64.yml.gz": {Size: 999, MD5: "bad", SHA256: "bad"}, + } + + downloader = http.NewFakeDownloader() + downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components") + + err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, true) + c.Assert(err, IsNil) + c.Check(s.repo.AppStreamFiles, HasLen, 1) + c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "") +} + type RemoteRepoCollectionSuite struct { PackageListMixinSuite db database.Storage @@ -706,7 +815,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) { _, err := s.collection.ByName("yandex") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Add(repo), IsNil) c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists") @@ -724,7 +833,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { _, err := s.collection.ByUUID("some-uuid") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Add(repo), IsNil) r, err := s.collection.ByUUID(repo.UUID) @@ -738,7 +847,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { } func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Update(repo), IsNil) collection := NewRemoteRepoCollection(s.db) @@ -759,7 +868,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { } func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo) count := 0 @@ -781,10 +890,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { } func (s *RemoteRepoCollectionSuite) TestDrop(c *C) { - repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo1) - repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false) + repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo2) r1, _ := s.collection.ByUUID(repo1.UUID) diff --git a/deb/snapshot.go b/deb/snapshot.go index 74f8865db3..f79a50d237 100644 --- a/deb/snapshot.go +++ b/deb/snapshot.go @@ -40,6 +40,9 @@ type Snapshot struct { NotAutomatic string ButAutomaticUpgrades string + // AppStream files: relative path → pool path (pass-through from mirror) + AppStreamFiles map[string]string `json:",omitempty"` + packageRefs *PackageRefList } @@ -59,6 +62,7 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error) Origin: repo.Meta["Origin"], NotAutomatic: repo.Meta["NotAutomatic"], ButAutomaticUpgrades: repo.Meta["ButAutomaticUpgrades"], + AppStreamFiles: repo.AppStreamFiles, packageRefs: repo.packageRefs, }, nil } @@ -94,14 +98,28 @@ func NewSnapshotFromRefList(name string, sources []*Snapshot, list *PackageRefLi sourceUUIDs[i] = sources[i].UUID } + // Merge AppStreamFiles from all source snapshots + var mergedAppStream map[string]string + for _, source := range sources { + if len(source.AppStreamFiles) > 0 { + if mergedAppStream == nil { + mergedAppStream = make(map[string]string) + } + for k, v := range source.AppStreamFiles { + mergedAppStream[k] = v + } + } + } + return &Snapshot{ - UUID: uuid.NewString(), - Name: name, - CreatedAt: time.Now(), - SourceKind: "snapshot", - SourceIDs: sourceUUIDs, - Description: description, - packageRefs: list, + UUID: uuid.NewString(), + Name: name, + CreatedAt: time.Now(), + SourceKind: "snapshot", + SourceIDs: sourceUUIDs, + Description: description, + AppStreamFiles: mergedAppStream, + packageRefs: list, } } diff --git a/deb/snapshot_test.go b/deb/snapshot_test.go index cbd5285c3c..d73432fe8e 100644 --- a/deb/snapshot_test.go +++ b/deb/snapshot_test.go @@ -19,7 +19,7 @@ var _ = Suite(&SnapshotSuite{}) func (s *SnapshotSuite) SetUpTest(c *C) { s.SetUpPackages() - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) s.repo.packageRefs = s.reflist } @@ -101,6 +101,62 @@ func (s *SnapshotSuite) TestEncodeDecode(c *C) { c.Assert(snapshot2.packageRefs, IsNil) } +func (s *SnapshotSuite) TestSnapshotFromRepositoryAppStream(c *C) { + s.repo.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": "ab/cd/Components-amd64.yml.gz", + "main/dep11/icons-48x48.tar.gz": "ef/gh/icons-48x48.tar.gz", + } + snapshot, err := NewSnapshotFromRepository("snap-as", s.repo) + c.Assert(err, IsNil) + c.Check(snapshot.AppStreamFiles, DeepEquals, s.repo.AppStreamFiles) + + s.repo.AppStreamFiles = nil + snapshot2, err := NewSnapshotFromRepository("snap-no-as", s.repo) + c.Assert(err, IsNil) + c.Check(snapshot2.AppStreamFiles, IsNil) +} + +func (s *SnapshotSuite) TestSnapshotFromRefListAppStreamMerge(c *C) { + snap1, _ := NewSnapshotFromRepository("snap1", s.repo) + snap1.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": "aa/bb/Components-amd64.yml.gz", + "main/dep11/icons-48x48.tar.gz": "cc/dd/icons-48x48.tar.gz", + } + + snap2, _ := NewSnapshotFromRepository("snap2", s.repo) + snap2.AppStreamFiles = map[string]string{ + "contrib/dep11/Components-amd64.yml.gz": "ee/ff/Components-amd64.yml.gz", + "main/dep11/Components-amd64.yml.gz": "xx/yy/Components-amd64.yml.gz", + } + + merged := NewSnapshotFromRefList("merged", []*Snapshot{snap1, snap2}, s.reflist, "Merged") + + c.Check(len(merged.AppStreamFiles), Equals, 3) + c.Check(merged.AppStreamFiles["main/dep11/icons-48x48.tar.gz"], Equals, "cc/dd/icons-48x48.tar.gz") + c.Check(merged.AppStreamFiles["contrib/dep11/Components-amd64.yml.gz"], Equals, "ee/ff/Components-amd64.yml.gz") + c.Check(merged.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Equals, "xx/yy/Components-amd64.yml.gz") + + snap3, _ := NewSnapshotFromRepository("snap3", s.repo) + snap3.AppStreamFiles = nil + snap4, _ := NewSnapshotFromRepository("snap4", s.repo) + snap4.AppStreamFiles = nil + merged2 := NewSnapshotFromRefList("merged2", []*Snapshot{snap3, snap4}, s.reflist, "Merged2") + c.Check(merged2.AppStreamFiles, IsNil) +} + +func (s *SnapshotSuite) TestEncodeDecodeAppStream(c *C) { + snapshot, _ := NewSnapshotFromRepository("snap-as-enc", s.repo) + snapshot.AppStreamFiles = map[string]string{ + "main/dep11/Components-amd64.yml.gz": "ab/cd/Components-amd64.yml.gz", + "main/dep11/icons-48x48.tar.gz": "ef/gh/icons-48x48.tar.gz", + } + + decoded := &Snapshot{} + c.Assert(decoded.Decode(snapshot.Encode()), IsNil) + c.Check(decoded.Name, Equals, snapshot.Name) + c.Check(decoded.AppStreamFiles, DeepEquals, snapshot.AppStreamFiles) +} + type SnapshotCollectionSuite struct { PackageListMixinSuite db database.Storage @@ -118,11 +174,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) { s.collection = NewSnapshotCollection(s.db) s.SetUpPackages() - s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) s.repo1.packageRefs = s.reflist s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1) - s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) + s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false) s.repo2.packageRefs = s.reflist s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2) @@ -223,7 +279,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) { c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1}) c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2}) - repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) + repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false) c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil)) } diff --git a/man/aptly.1 b/man/aptly.1 index b777d45261..745b07590d 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -615,6 +615,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time max download tries till process fails with download error . .TP +\-\fBwith\-appstream\fR +download AppStream (DEP\-11) metadata +. +.TP \-\fBwith\-installer\fR download additional not packaged installer files . @@ -790,6 +794,10 @@ disable verification of Release file signatures gpg keyring to use when verifying Release file (could be specified multiple times) . .TP +\-\fBwith\-appstream\fR +download AppStream (DEP\-11) metadata +. +.TP \-\fBwith\-installer\fR download additional not packaged installer files . diff --git a/system/t03_help/MirrorCreateHelpTest_gold b/system/t03_help/MirrorCreateHelpTest_gold index c0cc388692..6abb4ccf55 100644 --- a/system/t03_help/MirrorCreateHelpTest_gold +++ b/system/t03_help/MirrorCreateHelpTest_gold @@ -29,6 +29,7 @@ Options: -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) -max-tries=1: max download tries till process fails with download error + -with-appstream: download AppStream (DEP-11) metadata -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) diff --git a/system/t03_help/MirrorCreateTest_gold b/system/t03_help/MirrorCreateTest_gold index 30bcfd86b0..e594655c90 100644 --- a/system/t03_help/MirrorCreateTest_gold +++ b/system/t03_help/MirrorCreateTest_gold @@ -20,6 +20,7 @@ Options: -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) -max-tries=1: max download tries till process fails with download error + -with-appstream: download AppStream (DEP-11) metadata -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) diff --git a/system/t03_help/WrongFlagTest_gold b/system/t03_help/WrongFlagTest_gold index fb928b1ec8..f265a8f43f 100644 --- a/system/t03_help/WrongFlagTest_gold +++ b/system/t03_help/WrongFlagTest_gold @@ -21,6 +21,7 @@ Options: -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) -max-tries=1: max download tries till process fails with download error + -with-appstream: download AppStream (DEP-11) metadata -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) diff --git a/system/t04_mirror/CreateMirror11Test_mirror_show b/system/t04_mirror/CreateMirror11Test_mirror_show index b2c40eb698..1e2cfb9753 100644 --- a/system/t04_mirror/CreateMirror11Test_mirror_show +++ b/system/t04_mirror/CreateMirror11Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror13Test_mirror_show b/system/t04_mirror/CreateMirror13Test_mirror_show index 61e380fda4..fcb883e90d 100644 --- a/system/t04_mirror/CreateMirror13Test_mirror_show +++ b/system/t04_mirror/CreateMirror13Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror14Test_mirror_show b/system/t04_mirror/CreateMirror14Test_mirror_show index 38056b541d..a6035d2e86 100644 --- a/system/t04_mirror/CreateMirror14Test_mirror_show +++ b/system/t04_mirror/CreateMirror14Test_mirror_show @@ -5,6 +5,7 @@ Components: Architectures: all, amd64, i386 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror17Test_mirror_show b/system/t04_mirror/CreateMirror17Test_mirror_show index 5f233f2985..8db84df699 100644 --- a/system/t04_mirror/CreateMirror17Test_mirror_show +++ b/system/t04_mirror/CreateMirror17Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: i386 Download Sources: yes Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror18Test_mirror_show b/system/t04_mirror/CreateMirror18Test_mirror_show index 750161b0d3..2f0227788c 100644 --- a/system/t04_mirror/CreateMirror18Test_mirror_show +++ b/system/t04_mirror/CreateMirror18Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, armel, i386, powerpc Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror19Test_mirror_show b/system/t04_mirror/CreateMirror19Test_mirror_show index 5bcaf744a3..c3ddc24348 100644 --- a/system/t04_mirror/CreateMirror19Test_mirror_show +++ b/system/t04_mirror/CreateMirror19Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: i386 Download Sources: yes Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror1Test_mirror_show b/system/t04_mirror/CreateMirror1Test_mirror_show index 3acf9b5b73..8f62fc5f74 100644 --- a/system/t04_mirror/CreateMirror1Test_mirror_show +++ b/system/t04_mirror/CreateMirror1Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror21Test_mirror_show b/system/t04_mirror/CreateMirror21Test_mirror_show index 2eed39becf..41778e4f58 100644 --- a/system/t04_mirror/CreateMirror21Test_mirror_show +++ b/system/t04_mirror/CreateMirror21Test_mirror_show @@ -5,6 +5,7 @@ Components: Architectures: all Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror22Test_mirror_show b/system/t04_mirror/CreateMirror22Test_mirror_show index 2e8ddb9d16..6388e9f5da 100644 --- a/system/t04_mirror/CreateMirror22Test_mirror_show +++ b/system/t04_mirror/CreateMirror22Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386 Download Sources: no Download .udebs: no +Download AppStream: no Filter: nginx | Priority (required) Filter With Deps: no Last update: never diff --git a/system/t04_mirror/CreateMirror25Test_mirror_show b/system/t04_mirror/CreateMirror25Test_mirror_show index 0b2c675ed6..e14bb34cb5 100644 --- a/system/t04_mirror/CreateMirror25Test_mirror_show +++ b/system/t04_mirror/CreateMirror25Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: i386 Download Sources: no Download .udebs: yes +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror27Test_mirror_show b/system/t04_mirror/CreateMirror27Test_mirror_show index 97f5625542..8b832ed5fd 100644 --- a/system/t04_mirror/CreateMirror27Test_mirror_show +++ b/system/t04_mirror/CreateMirror27Test_mirror_show @@ -5,6 +5,7 @@ Components: openmanage/740 Architectures: amd64, i386 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror2Test_mirror_show b/system/t04_mirror/CreateMirror2Test_mirror_show index e17e961976..b8061dc580 100644 --- a/system/t04_mirror/CreateMirror2Test_mirror_show +++ b/system/t04_mirror/CreateMirror2Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror32Test_mirror_show b/system/t04_mirror/CreateMirror32Test_mirror_show index 6cd09c53aa..2880a495cf 100644 --- a/system/t04_mirror/CreateMirror32Test_mirror_show +++ b/system/t04_mirror/CreateMirror32Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror33Test_mirror_show b/system/t04_mirror/CreateMirror33Test_mirror_show index 24e2493890..2a0b05df6c 100644 --- a/system/t04_mirror/CreateMirror33Test_mirror_show +++ b/system/t04_mirror/CreateMirror33Test_mirror_show @@ -5,6 +5,7 @@ Components: Architectures: amd64 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror35Test_mirror_show b/system/t04_mirror/CreateMirror35Test_mirror_show index 3ae1920a4b..83bc7c1c93 100644 --- a/system/t04_mirror/CreateMirror35Test_mirror_show +++ b/system/t04_mirror/CreateMirror35Test_mirror_show @@ -5,6 +5,7 @@ Components: Architectures: amd64 Download Sources: no Download .udebs: no +Download AppStream: no Filter: cuda-12-6 (= 12.6.2-1) Filter With Deps: yes Last update: never diff --git a/system/t04_mirror/CreateMirror36Test_mirror_show b/system/t04_mirror/CreateMirror36Test_mirror_show index c6188417cc..84fe9226e9 100644 --- a/system/t04_mirror/CreateMirror36Test_mirror_show +++ b/system/t04_mirror/CreateMirror36Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386 Download Sources: no Download .udebs: no +Download AppStream: no Filter: nginx | Priority (required) Filter With Deps: no Last update: never diff --git a/system/t04_mirror/CreateMirror37Test_mirror_show b/system/t04_mirror/CreateMirror37Test_mirror_show index aa2750a297..94951ab519 100644 --- a/system/t04_mirror/CreateMirror37Test_mirror_show +++ b/system/t04_mirror/CreateMirror37Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386 Download Sources: no Download .udebs: no +Download AppStream: no Filter: nginx | Priority (required) Filter With Deps: no Last update: never diff --git a/system/t04_mirror/CreateMirror38Test_gold b/system/t04_mirror/CreateMirror38Test_gold new file mode 100644 index 0000000000..fbfac77525 --- /dev/null +++ b/system/t04_mirror/CreateMirror38Test_gold @@ -0,0 +1,4 @@ +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release + +Mirror [mirror38]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch [appstream] successfully added. +You can run 'aptly mirror update mirror38' to download repository contents. diff --git a/system/t04_mirror/CreateMirror38Test_mirror_show b/system/t04_mirror/CreateMirror38Test_mirror_show new file mode 100644 index 0000000000..0cb4bb5352 --- /dev/null +++ b/system/t04_mirror/CreateMirror38Test_mirror_show @@ -0,0 +1,23 @@ +Name: mirror38 +Archive Root URL: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ +Distribution: stretch +Components: main, contrib, non-free +Architectures: amd64 +Download Sources: no +Download .udebs: no +Download AppStream: yes +Last update: never + +Information from release file: +Acquire-By-Hash: yes +Architectures: amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x +Changelogs: http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog +Codename: stretch +Components: main contrib non-free +Date: Sat, 14 Aug 2021 07:42:00 UTC +Description: Debian 9.13 Released 18 July 2020 + +Label: Debian +Origin: Debian +Suite: oldoldstable +Version: 9.13 diff --git a/system/t04_mirror/CreateMirror3Test_mirror_show b/system/t04_mirror/CreateMirror3Test_mirror_show index f016c6efb3..f593616773 100644 --- a/system/t04_mirror/CreateMirror3Test_mirror_show +++ b/system/t04_mirror/CreateMirror3Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib Architectures: i386, amd64 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror7Test_mirror_show b/system/t04_mirror/CreateMirror7Test_mirror_show index a62cfb67a2..496885dbe4 100644 --- a/system/t04_mirror/CreateMirror7Test_mirror_show +++ b/system/t04_mirror/CreateMirror7Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib Architectures: i386, amd64 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/CreateMirror9Test_mirror_show b/system/t04_mirror/CreateMirror9Test_mirror_show index 302a565c8c..84bda45394 100644 --- a/system/t04_mirror/CreateMirror9Test_mirror_show +++ b/system/t04_mirror/CreateMirror9Test_mirror_show @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/EditMirror11Test_gold b/system/t04_mirror/EditMirror11Test_gold new file mode 100644 index 0000000000..3f9345a288 --- /dev/null +++ b/system/t04_mirror/EditMirror11Test_gold @@ -0,0 +1 @@ +Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy [appstream] successfully updated. diff --git a/system/t04_mirror/EditMirror11Test_mirror_show b/system/t04_mirror/EditMirror11Test_mirror_show new file mode 100644 index 0000000000..16808beca8 --- /dev/null +++ b/system/t04_mirror/EditMirror11Test_mirror_show @@ -0,0 +1,21 @@ +Name: wheezy-main +Archive Root URL: http://mirror.yandex.ru/debian/ +Distribution: wheezy +Components: main +Architectures: i386, amd64 +Download Sources: no +Download .udebs: no +Download AppStream: yes +Number of packages: 56121 + +Information from release file: +Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc +Codename: wheezy +Components: main contrib non-free +Date: Sat, 26 Apr 2014 09:27:11 UTC +Description: Debian 7.5 Released 26 April 2014 + +Label: Debian +Origin: Debian +Suite: stable +Version: 7.5 diff --git a/system/t04_mirror/EditMirror1Test_mirror_show b/system/t04_mirror/EditMirror1Test_mirror_show index f934ffc92b..7a57236b1a 100644 --- a/system/t04_mirror/EditMirror1Test_mirror_show +++ b/system/t04_mirror/EditMirror1Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: i386, amd64 Download Sources: yes Download .udebs: no +Download AppStream: no Filter: nginx Filter With Deps: yes Number of packages: 56121 diff --git a/system/t04_mirror/EditMirror3Test_mirror_show b/system/t04_mirror/EditMirror3Test_mirror_show index d3348f0c6f..4c362234e6 100644 --- a/system/t04_mirror/EditMirror3Test_mirror_show +++ b/system/t04_mirror/EditMirror3Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: i386, amd64 Download Sources: no Download .udebs: no +Download AppStream: no Number of packages: 56121 Information from release file: diff --git a/system/t04_mirror/EditMirror5Test_mirror_show b/system/t04_mirror/EditMirror5Test_mirror_show index c6dfb1fbf8..c9dc38853e 100644 --- a/system/t04_mirror/EditMirror5Test_mirror_show +++ b/system/t04_mirror/EditMirror5Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386 Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/EditMirror6Test_mirror_show b/system/t04_mirror/EditMirror6Test_mirror_show index ac42dcfc04..1c8efb34af 100644 --- a/system/t04_mirror/EditMirror6Test_mirror_show +++ b/system/t04_mirror/EditMirror6Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: amd64, i386 Download Sources: no Download .udebs: no +Download AppStream: no Information from release file: Acquire-By-Hash: yes diff --git a/system/t04_mirror/EditMirror8Test_mirror_show b/system/t04_mirror/EditMirror8Test_mirror_show index 3aa031310a..3771269232 100644 --- a/system/t04_mirror/EditMirror8Test_mirror_show +++ b/system/t04_mirror/EditMirror8Test_mirror_show @@ -5,6 +5,7 @@ Components: main Architectures: i386, amd64 Download Sources: no Download .udebs: yes +Download AppStream: no Number of packages: 56121 Information from release file: diff --git a/system/t04_mirror/ListMirror6Test_gold b/system/t04_mirror/ListMirror6Test_gold index 18cfd1c3c9..9dae0a1bc8 100644 --- a/system/t04_mirror/ListMirror6Test_gold +++ b/system/t04_mirror/ListMirror6Test_gold @@ -42,7 +42,8 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false }, { "Name": "mirror2", @@ -85,7 +86,8 @@ "SkipArchitectureCheck": false, "DownloadSources": true, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false }, { "Name": "mirror3", @@ -119,7 +121,8 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false }, { "Name": "mirror4", @@ -148,6 +151,7 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false } ] diff --git a/system/t04_mirror/ShowMirror1Test_gold b/system/t04_mirror/ShowMirror1Test_gold index 3acf9b5b73..8f62fc5f74 100644 --- a/system/t04_mirror/ShowMirror1Test_gold +++ b/system/t04_mirror/ShowMirror1Test_gold @@ -5,6 +5,7 @@ Components: main, contrib, non-free Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x Download Sources: no Download .udebs: no +Download AppStream: no Last update: never Information from release file: diff --git a/system/t04_mirror/ShowMirror3Test_gold b/system/t04_mirror/ShowMirror3Test_gold index 14e47a2874..7d83da7683 100644 --- a/system/t04_mirror/ShowMirror3Test_gold +++ b/system/t04_mirror/ShowMirror3Test_gold @@ -5,6 +5,7 @@ Components: contrib Architectures: i386, amd64 Download Sources: no Download .udebs: no +Download AppStream: no Number of packages: 325 Information from release file: diff --git a/system/t04_mirror/ShowMirror4Test_gold b/system/t04_mirror/ShowMirror4Test_gold index 48ff3286c5..95ae861a8f 100644 --- a/system/t04_mirror/ShowMirror4Test_gold +++ b/system/t04_mirror/ShowMirror4Test_gold @@ -5,6 +5,7 @@ Components: main Architectures: amd64, arm64, armel, armhf, i386 Download Sources: no Download .udebs: no +Download AppStream: no Filter: nginx | Priority (required) Filter With Deps: yes Last update: never diff --git a/system/t04_mirror/ShowMirror5Test_gold b/system/t04_mirror/ShowMirror5Test_gold index d8ba1780ca..79e5af4cff 100644 --- a/system/t04_mirror/ShowMirror5Test_gold +++ b/system/t04_mirror/ShowMirror5Test_gold @@ -41,5 +41,6 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false } diff --git a/system/t04_mirror/ShowMirror7Test_gold b/system/t04_mirror/ShowMirror7Test_gold index d454a9599f..3dbc910e30 100644 --- a/system/t04_mirror/ShowMirror7Test_gold +++ b/system/t04_mirror/ShowMirror7Test_gold @@ -31,6 +31,7 @@ "DownloadSources": false, "DownloadUdebs": false, "DownloadInstaller": false, + "DownloadAppStream": false, "Packages": [ "alien-arena-server_7.53+dfsg-3_amd64", "alien-arena-server_7.53+dfsg-3_i386", diff --git a/system/t04_mirror/ShowMirror8Test_gold b/system/t04_mirror/ShowMirror8Test_gold index 9d3f70c5db..1b8de6013c 100644 --- a/system/t04_mirror/ShowMirror8Test_gold +++ b/system/t04_mirror/ShowMirror8Test_gold @@ -33,5 +33,6 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false } diff --git a/system/t04_mirror/UpdateMirror28Test_gold b/system/t04_mirror/UpdateMirror28Test_gold new file mode 100644 index 0000000000..eba944c5b7 --- /dev/null +++ b/system/t04_mirror/UpdateMirror28Test_gold @@ -0,0 +1,26 @@ +Downloading: ${url}dists/hardy/Release +Downloading & parsing package files... +Downloading: ${url}dists/hardy/main/binary-amd64/Packages.bz2 +Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.bz2 +Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.bz2... +Download Error: ${url}dists/hardy/main/binary-amd64/Packages.bz2 +Downloading: ${url}dists/hardy/main/binary-amd64/Packages.gz +Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.gz +Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.gz... +Download Error: ${url}dists/hardy/main/binary-amd64/Packages.gz +Downloading: ${url}dists/hardy/main/binary-amd64/Packages.xz +Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.xz +Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.xz... +Download Error: ${url}dists/hardy/main/binary-amd64/Packages.xz +Downloading: ${url}dists/hardy/main/binary-amd64/Packages +Downloading AppStream metadata... +Downloading AppStream file main/dep11/Components-amd64.yml.gz... +Downloading: ${url}dists/hardy/main/dep11/Components-amd64.yml.gz +Downloading AppStream file main/dep11/icons-48x48.tar.gz... +Downloading: ${url}dists/hardy/main/dep11/icons-48x48.tar.gz +Building download queue... +Download queue: 1 items (30 B) +Downloading: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb +WARNING: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb: sha1 hash mismatch "8d3a014000038725d6daf8771b42a0784253688f" != "66b27417d37e024c46526c2f6d358a754fc552f3" + +Mirror `appstream-test` has been updated successfully. diff --git a/system/t04_mirror/UpdateMirror28Test_mirror_show b/system/t04_mirror/UpdateMirror28Test_mirror_show new file mode 100644 index 0000000000..cc49374631 --- /dev/null +++ b/system/t04_mirror/UpdateMirror28Test_mirror_show @@ -0,0 +1,21 @@ +Name: appstream-test +Archive Root URL: ${url} +Distribution: hardy +Components: main +Architectures: amd64 +Download Sources: no +Download .udebs: no +Download AppStream: yes +Number of packages: 1 + +Information from release file: +Architectures: amd64 +Codename: hardy +Components: main +Date: Sat, 19 Oct 2013 13:54:21 UTC +Description: Debian 6.0.8 Released 19 October 2013 + +Label: failure +Origin: test +Suite: test +Version: 6.0.8 diff --git a/system/t04_mirror/create.py b/system/t04_mirror/create.py index 351bb20e84..9c74bf3fc4 100644 --- a/system/t04_mirror/create.py +++ b/system/t04_mirror/create.py @@ -533,3 +533,14 @@ def removeDates(s): self.check_output() self.check_cmd_output("aptly mirror show mirror37", "mirror_show", match_prepare=removeDates) + + +class CreateMirror38Test(BaseTest): + """ + create mirror: mirror with appstream enabled + """ + runCmd = "aptly -architectures=amd64 mirror create -ignore-signatures -with-appstream mirror38 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch" + + def check(self): + self.check_output() + self.check_cmd_output("aptly mirror show mirror38", "mirror_show") diff --git a/system/t04_mirror/edit.py b/system/t04_mirror/edit.py index 09a5b18f32..b835748287 100644 --- a/system/t04_mirror/edit.py +++ b/system/t04_mirror/edit.py @@ -114,3 +114,16 @@ class EditMirror10Test(BaseTest): """ fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"] runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10" + + +class EditMirror11Test(BaseTest): + """ + edit mirror: enable appstream + """ + fixtureDB = True + runCmd = "aptly mirror edit -with-appstream wheezy-main" + + def check(self): + self.check_output() + self.check_cmd_output("aptly mirror show wheezy-main", "mirror_show", + match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s)) diff --git a/system/t04_mirror/test_release2/dists/hardy/Release b/system/t04_mirror/test_release2/dists/hardy/Release index a0c428a54a..cea3344331 100644 --- a/system/t04_mirror/test_release2/dists/hardy/Release +++ b/system/t04_mirror/test_release2/dists/hardy/Release @@ -9,3 +9,11 @@ Components: main Description: Debian 6.0.8 Released 19 October 2013 MD5Sum: 846549680001f5c632b6ee8e0f183825 827 main/binary-amd64/Packages + f72bdc4905a2d68a7a4b85544a7135c3 92 main/dep11/Components-amd64.yml.gz + 26dce1ff9274a848f1633737936044ed 170 main/dep11/icons-48x48.tar.gz +SHA1: + 2a4d420955106ebf1765153341ba4581680f62ce 92 main/dep11/Components-amd64.yml.gz + 2f14d886f1e798b899184e7ff1ef2d2f27cd59df 170 main/dep11/icons-48x48.tar.gz +SHA256: + 1245da5002d7e96c3b3133671e9267f69eea41ee4e3f3b8a0dcb6df15e5b51b7 92 main/dep11/Components-amd64.yml.gz + bdd3f75f27c2eaf78e074142ee2270809d5f16b26cc64c60df7cb38ae0f38516 170 main/dep11/icons-48x48.tar.gz diff --git a/system/t04_mirror/test_release2/dists/hardy/main/dep11/Components-amd64.yml.gz b/system/t04_mirror/test_release2/dists/hardy/main/dep11/Components-amd64.yml.gz new file mode 100644 index 0000000000..82e07cdeaa Binary files /dev/null and b/system/t04_mirror/test_release2/dists/hardy/main/dep11/Components-amd64.yml.gz differ diff --git a/system/t04_mirror/test_release2/dists/hardy/main/dep11/icons-48x48.tar.gz b/system/t04_mirror/test_release2/dists/hardy/main/dep11/icons-48x48.tar.gz new file mode 100644 index 0000000000..ef0410fb8e Binary files /dev/null and b/system/t04_mirror/test_release2/dists/hardy/main/dep11/icons-48x48.tar.gz differ diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index fb44dd7e85..c047176792 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -482,3 +482,23 @@ class UpdateMirror27Test(BaseTest): runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail" outputMatchPrepare = filterOutRedirects expectedCode = 1 + + +class UpdateMirror28Test(BaseTest): + """ + update mirrors: update with appstream + """ + fixtureCmds = [ + "aptly mirror create --ignore-signatures -with-appstream appstream-test ${url} hardy main", + ] + fixtureWebServer = "test_release2" + configOverride = {"downloadRetries": 0} + runCmd = "aptly mirror update -ignore-checksums --ignore-signatures appstream-test" + + def gold_processor(self, gold): + return string.Template(gold).substitute({'url': self.webServerUrl}) + + def check(self): + self.check_output() + self.check_cmd_output("aptly mirror show appstream-test", "mirror_show", + match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s)) diff --git a/system/t05_snapshot/ShowSnapshot4Test_gold b/system/t05_snapshot/ShowSnapshot4Test_gold index 11ea4f8739..c814ee0058 100644 --- a/system/t05_snapshot/ShowSnapshot4Test_gold +++ b/system/t05_snapshot/ShowSnapshot4Test_gold @@ -34,7 +34,8 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false } ], "Description": "Snapshot from mirror [wheezy-non-free]: http://mirror.yandex.ru/debian/ wheezy", diff --git a/system/t05_snapshot/ShowSnapshot5Test_gold b/system/t05_snapshot/ShowSnapshot5Test_gold index 6d6a42c453..f6b6a9417a 100644 --- a/system/t05_snapshot/ShowSnapshot5Test_gold +++ b/system/t05_snapshot/ShowSnapshot5Test_gold @@ -36,7 +36,8 @@ "SkipArchitectureCheck": false, "DownloadSources": false, "DownloadUdebs": false, - "DownloadInstaller": false + "DownloadInstaller": false, + "DownloadAppStream": false } ], "Packages": [ diff --git a/system/t06_publish/PublishSnapshot44Test_gold b/system/t06_publish/PublishSnapshot44Test_gold new file mode 100644 index 0000000000..740f915476 --- /dev/null +++ b/system/t06_publish/PublishSnapshot44Test_gold @@ -0,0 +1,14 @@ +Loading packages... +Generating metadata files and linking package files... +[!] Failed to generate package contents: unable to read .deb archive from amanda-client_3.3.1-3~bpo60+1_amd64.deb: ar: missing global header +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Snapshot snap-appstream has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ hardy main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index b35d6f3440..b338032f07 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1439,3 +1439,24 @@ def check(self): self.check_file_contents( 'public/dists/maverick/Release', 'release', match_prepare=strip_processor) + + +class PublishSnapshot44Test(BaseTest): + """ + publish snapshot: mirror with appstream + """ + fixtureWebServer = "../t04_mirror/test_release2" + fixtureGpg = True + fixtureCmds = [ + "aptly mirror create --ignore-signatures -with-appstream -architectures=amd64 appstream-test ${url} hardy main", + "aptly mirror update -ignore-checksums --ignore-signatures appstream-test", + "aptly snapshot create snap-appstream from mirror appstream-test", + ] + runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap-appstream" + gold_processor = BaseTest.expand_environ + configOverride = {"downloadRetries": 0} + + def check(self): + super(PublishSnapshot44Test, self).check() + self.check_exists('public/dists/hardy/main/dep11/Components-amd64.yml.gz') + self.check_exists('public/dists/hardy/main/dep11/icons-48x48.tar.gz') diff --git a/system/t08_db/CleanupDB13Test_gold b/system/t08_db/CleanupDB13Test_gold new file mode 100644 index 0000000000..138adc2947 --- /dev/null +++ b/system/t08_db/CleanupDB13Test_gold @@ -0,0 +1,7 @@ +Loading mirrors, local repos, snapshots and published repos... +Loading list of all packages... +Deleting unreferenced packages (0)... +Building list of files referenced by packages... +Building list of files in package pool... +Deleting unreferenced files (0)... +Compacting database... diff --git a/system/t08_db/CleanupDB13Test_publish b/system/t08_db/CleanupDB13Test_publish new file mode 100644 index 0000000000..740f915476 --- /dev/null +++ b/system/t08_db/CleanupDB13Test_publish @@ -0,0 +1,14 @@ +Loading packages... +Generating metadata files and linking package files... +[!] Failed to generate package contents: unable to read .deb archive from amanda-client_3.3.1-3~bpo60+1_amd64.deb: ar: missing global header +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Snapshot snap-appstream has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ hardy main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t08_db/cleanup.py b/system/t08_db/cleanup.py index c0633426ba..ff8bef0d31 100644 --- a/system/t08_db/cleanup.py +++ b/system/t08_db/cleanup.py @@ -150,3 +150,27 @@ class CleanupDB12Test(BaseTest): "aptly mirror drop gnuplot-maverick", ] runCmd = "aptly db cleanup -verbose -dry-run" + + +class CleanupDB13Test(BaseTest): + """ + cleanup db: appstream files survive cleanup + """ + fixtureWebServer = "../t04_mirror/test_release2" + fixtureGpg = True + configOverride = {"downloadRetries": 0} + fixtureCmds = [ + "aptly mirror create --ignore-signatures -with-appstream -architectures=amd64 appstream-test ${url} hardy main", + "aptly mirror update -ignore-checksums --ignore-signatures appstream-test", + "aptly snapshot create snap-appstream from mirror appstream-test", + ] + runCmd = "aptly db cleanup" + + def check(self): + self.check_output() + # verify appstream files survive cleanup by publishing the snapshot + self.check_cmd_output( + "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap-appstream", + "publish", match_prepare=self.expand_environ) + self.check_exists('public/dists/hardy/main/dep11/Components-amd64.yml.gz') + self.check_exists('public/dists/hardy/main/dep11/icons-48x48.tar.gz')