Add appstream support

AUTHORS

@@ -81,3 +81,4 @@ List of contributors, in chronological order:
 * Brian Witt (https://github.com/bwitt)
 * Ales Bregar (https://github.com/abregar)
 * Tim Foerster (https://github.com/tonobo)
+* Philip Cramer (https://github.com/PhilipCramer)

api/mirror.go

@@ -72,6 +72,8 @@ type mirrorCreateParams struct {
 	DownloadUdebs bool `                     json:"DownloadUdebs"`
 	// Set "true" to mirror installer files
 	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to mirror AppStream (DEP-11) metadata
+	DownloadAppStream bool `                 json:"DownloadAppStream"`
 	// Set "true" to include dependencies of matching packages when filtering
 	FilterWithDeps bool `                    json:"FilterWithDeps"`
 	// Set "true" to skip if the given components are in the Release file
@@ -123,7 +125,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	}
 
 	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
-		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream)
 
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
@@ -421,6 +423,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		if remote.DownloadAppStream && !remote.IsFlat() {
+			err = remote.DownloadAppStreamFiles(out, downloader,
+				context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		if remote.Filter != "" {
 			var filterQuery deb.PackageQuery
 

api/mirror_test.go

@@ -26,6 +26,21 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
 	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
 }
 
+func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name":              "test-flat-appstream",
+		"ArchiveURL":        "http://example.com/repo/",
+		"Distribution":      "./",
+		"DownloadAppStream":  true,
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
+}
+
 func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.ExpectFailure("Need to mock downloads")
 	body, err := json.Marshal(gin.H{

cmd/db_cleanup.go

@@ -26,6 +26,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
+	referencedAppStreamFiles := []string{}
 
 	// used only in verbose mode to report package use source
 	packageRefSources := map[string][]string{}
@@ -55,6 +56,10 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 			}
 		}
 
+		for _, poolPath := range repo.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -118,6 +123,11 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 				return nil
 			})
 		}
+
+		for _, poolPath := range snapshot.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -236,6 +246,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
 	sort.Strings(referencedFiles)
 	context.Progress().ShutdownBar()
 

cmd/mirror_create.go

@@ -20,6 +20,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
 	ignoreSignatures := context.Config().GpgDisableVerify
 	if context.Flags().IsSet("ignore-signatures") {
 		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
@@ -41,7 +42,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs, downloadInstaller)
+		downloadSources, downloadUdebs, downloadInstaller, downloadAppStream)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -100,6 +101,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_edit.go

@@ -35,6 +35,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-appstream":
+			repo.DownloadAppStream = flag.Value.Get().(bool)
 		case "with-installer":
 			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
@@ -53,6 +55,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
 	}
 
+	if repo.IsFlat() && repo.DownloadAppStream {
+		return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -107,6 +113,7 @@ Example:
 	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_show.go

@@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	downloadAppStream := No
+	if repo.DownloadAppStream {
+		downloadAppStream = Yes
+	}
+	fmt.Printf("Download AppStream: %s\n", downloadAppStream)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := No

cmd/mirror_update.go

@@ -64,6 +64,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.DownloadAppStream && !repo.IsFlat() {
+		context.Progress().Printf("Downloading AppStream metadata...\n")
+		err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
+			context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	if repo.Filter != "" {
 		context.Progress().Printf("Applying filter...\n")
 		var filterQuery deb.PackageQuery

completion.d/_aptly

@@ -185,6 +185,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
                         ;;
                     list)
@@ -224,6 +225,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     search)

completion.d/aptly

@@ -203,15 +203,15 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
           ;;
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

deb/package_test.go

@@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	downloader := http.NewFakeDownloader()
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")

deb/publish.go

@@ -1053,6 +1053,38 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			}
 		}
 
+		// Pass-through AppStream (DEP-11) files from snapshots
+		for component, item := range p.sourceItems {
+			if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 {
+				continue
+			}
+
+			prefix := component + "/"
+			for relPath, poolPath := range item.snapshot.AppStreamFiles {
+				if !strings.HasPrefix(relPath, prefix) {
+					continue
+				}
+				withinComponent := strings.TrimPrefix(relPath, prefix)
+
+				poolFile, err := packagePool.Open(poolPath)
+				if err != nil {
+					return fmt.Errorf("unable to open AppStream file from pool: %v", err)
+				}
+
+				bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter()
+				if err != nil {
+					poolFile.Close()
+					return fmt.Errorf("unable to generate AppStream index: %v", err)
+				}
+
+				_, err = bufio.NewReader(poolFile).WriteTo(bufWriter)
+				poolFile.Close()
+				if err != nil {
+					return fmt.Errorf("unable to write AppStream file: %v", err)
+				}
+			}
+		}
+
 		udebs := []bool{false}
 		if hadUdebs {
 			udebs = append(udebs, true)

deb/publish_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/ugorji/go/codec"
 
 	. "gopkg.in/check.v1"
@@ -115,7 +116,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	repo.packageRefs = s.reflist
 	_ = s.factory.RemoteRepoCollection().Add(repo)
 
@@ -425,6 +426,81 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedRepoSuite) TestPublishAppStream(c *C) {
+	// Components + icons
+	content1 := []byte("DEP-11 test content for Components-amd64.yml.gz")
+	tmpFile1 := filepath.Join(c.MkDir(), "Components-amd64.yml.gz")
+	c.Assert(os.WriteFile(tmpFile1, content1, 0644), IsNil)
+
+	checksums1 := utils.ChecksumInfo{Size: int64(len(content1))}
+	poolPath1, err := s.packagePool.Import(tmpFile1, "Components-amd64.yml.gz", &checksums1, false, s.cs)
+	c.Assert(err, IsNil)
+
+	content2 := []byte("DEP-11 icons tar data")
+	tmpFile2 := filepath.Join(c.MkDir(), "icons-48x48.tar.gz")
+	c.Assert(os.WriteFile(tmpFile2, content2, 0644), IsNil)
+
+	checksums2 := utils.ChecksumInfo{Size: int64(len(content2))}
+	poolPath2, err := s.packagePool.Import(tmpFile2, "icons-48x48.tar.gz", &checksums2, false, s.cs)
+	c.Assert(err, IsNil)
+
+	// Include contrib file that should be skipped
+	contribContent := []byte("DEP-11 contrib content")
+	tmpFile3 := filepath.Join(c.MkDir(), "Components-contrib.yml.gz")
+	c.Assert(os.WriteFile(tmpFile3, contribContent, 0644), IsNil)
+
+	checksums3 := utils.ChecksumInfo{Size: int64(len(contribContent))}
+	poolPath3, err := s.packagePool.Import(tmpFile3, "Components-contrib.yml.gz", &checksums3, false, s.cs)
+	c.Assert(err, IsNil)
+
+	s.snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz":    poolPath1,
+		"main/dep11/icons-48x48.tar.gz":         poolPath2,
+		"contrib/dep11/Components-amd64.yml.gz": poolPath3,
+	}
+
+	err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	// Both main files should exist
+	appstreamPath1 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/Components-amd64.yml.gz")
+	c.Check(appstreamPath1, PathExists)
+	actual1, err := os.ReadFile(appstreamPath1)
+	c.Assert(err, IsNil)
+	c.Check(actual1, DeepEquals, content1)
+
+	appstreamPath2 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/icons-48x48.tar.gz")
+	c.Check(appstreamPath2, PathExists)
+	actual2, err := os.ReadFile(appstreamPath2)
+	c.Assert(err, IsNil)
+	c.Check(actual2, DeepEquals, content2)
+
+	// Contrib file should not appear
+	contribPath := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/contrib/dep11/Components-amd64.yml.gz")
+	_, statErr := os.Stat(contribPath)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+
+	// Release file should reference AppStream files
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer rf.Close()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["MD5Sum"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
+	c.Check(st["SHA256"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
+
+	// Pool open error
+	s.snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": "nonexistent/pool/path",
+	}
+
+	err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*")
+}
+
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)