diff --git a/Taskfile.yml b/Taskfile.yml index ce12f46b..6a2bf54c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -302,52 +302,3 @@ tasks: desc: Clean all binaries cmds: - rm -rf ./build ./dist ./d8 - - release:tag: - desc: Tag current main branch state as new release. Provide version tag as argument (eg. task {{.TASK}} -- v1.2.3). - preconditions: - - sh: test $(git rev-parse --abbrev-ref HEAD) = "main" - msg: This can only be done on main branch - - sh: '[[ "{{.CLI_ARGS}}" =~ ^v\d+.\d+.\d+$ ]] || exit 1' - msg: Provide valid semver prefixed with "v" (eg. task {{.TASK}} -- v1.2.3) to be used as version - prompt: - - "Tag current main branch state as new release? ({{.CLI_ARGS}})" - - "Have you created Github release draft for {{.CLI_ARGS}} tag?" - cmds: - - git tag -s {{.CLI_ARGS}} -m 'Signed {{.CLI_ARGS}} release' - - git push origin {{.CLI_ARGS}} - - echo "Release tag created, now publish your Github release so that CI can upload build artifacts to it." - - release:sign: - desc: "Sign last version tag + origin/main and push signatures." - deps: [checkKubectl] - preconditions: - - sh: '[[ "{{.CLI_ARGS}}" =~ ^v\d+.\d+.\d+$ ]] || exit 1' - msg: Provide valid semver prefixed with "v" (eg. task {{.TASK}} -- v1.2.3) to be used as version - cmds: - - git fetch --tags -f - - git signatures pull - - | - for ref in {{.refs | default "$(git tag --sort=v:refname | tail -n1) origin/main"}}; do - echo "Signing $ref..." - git signatures add {{.CLI_ARGS}} $ref - git signatures show {{.CLI_ARGS}} $ref - done - - git signatures push - - release:publish-trdl-channels: - desc: Publish release channels to TRDL - deps: [checkKubectl] - preconditions: - - sh: test $(git rev-parse --abbrev-ref HEAD) = "main" - msg: This can only be done on main branch - prompt: - - "Have you updated trdl_channels.yaml with new versions?" - cmds: - - git add trdl_channels.yaml - - git commit -S -m 'Signed release channels' - - git push - - git fetch - - git signatures pull - - git signatures add origin/main - - git signatures push diff --git a/internal/mirror/cmd/push/flags.go b/internal/mirror/cmd/push/flags.go index 49a83d56..cebe7de8 100644 --- a/internal/mirror/cmd/push/flags.go +++ b/internal/mirror/cmd/push/flags.go @@ -63,6 +63,12 @@ func addFlags(flagSet *pflag.FlagSet) { "/modules", "Suffix to append to source repo path to locate modules.", ) + flagSet.StringArrayVar( + &Files, + "file", + nil, + "`Path` to a tar or chunked package to push. May be repeated to push multiple packages. Can be used instead of the bundle directory argument or combined with it.", + ) } func ParseEnvironmentVariables() { diff --git a/internal/mirror/cmd/push/push.go b/internal/mirror/cmd/push/push.go index ef27e21d..715053f9 100644 --- a/internal/mirror/cmd/push/push.go +++ b/internal/mirror/cmd/push/push.go @@ -56,6 +56,11 @@ var ( Insecure bool TLSSkipVerify bool ImagesBundlePath string + Files []string + + // Packages holds the resolved list of tar/chunked package archive paths to push, + // assembled from the bundle directory argument and/or the --file flag. + Packages []string MirrorTimeout time.Duration = -1 ) @@ -92,6 +97,7 @@ func NewCommand() *cobra.Command { Use: "push ", Short: "Copy Deckhouse Kubernetes Platform distribution to the third-party registry", Long: pushLong, + Args: cobra.RangeArgs(1, 2), ValidArgs: []string{"images-bundle-path", "registry"}, SilenceErrors: true, SilenceUsage: true, @@ -239,6 +245,7 @@ func (p *Pusher) executeNewPush() error { client, &mirror.PushServiceOptions{ BundleDir: p.pushParams.BundleDir, + Packages: Packages, WorkingDir: p.pushParams.WorkingDir, }, logger.Named("push"), diff --git a/internal/mirror/cmd/push/validation.go b/internal/mirror/cmd/push/validation.go index 6a08ae5b..36050cd1 100644 --- a/internal/mirror/cmd/push/validation.go +++ b/internal/mirror/cmd/push/validation.go @@ -32,12 +32,25 @@ import ( ) func parseAndValidateParameters(_ *cobra.Command, args []string) error { - if len(args) != 2 { - return errors.New("invalid number of arguments, expected 2") + // The registry is always the last argument. The bundle path is an optional + // first argument; when it is omitted, packages must be provided via --file. + var ( + registryArg string + bundleArg []string + ) + + switch len(args) { + case 1: + registryArg = args[0] + case 2: + bundleArg = args[:1] + registryArg = args[1] + default: + return errors.New("invalid number of arguments, expected with an optional bundle path before it") } var err error - if err = parseAndValidateRegistryURLArg(args); err != nil { + if err = parseAndValidateRegistryURLArg(registryArg); err != nil { return err } @@ -45,15 +58,45 @@ func parseAndValidateParameters(_ *cobra.Command, args []string) error { return err } - if err = validateImagesBundlePathArg(args); err != nil { + if err = resolvePackages(bundleArg); err != nil { return err } return nil } -func validateImagesBundlePathArg(args []string) error { - ImagesBundlePath = filepath.Clean(args[0]) +// resolvePackages builds the list of package archives to push from the optional +// bundle path argument and the --file flag, then sets the default temp dir. +func resolvePackages(bundleArg []string) error { + Packages = nil + + if len(bundleArg) == 1 { + if err := collectBundlePathPackages(bundleArg[0]); err != nil { + return err + } + } + + if err := collectFilesPackages(); err != nil { + return err + } + + if len(Packages) == 0 { + return errors.New("no packages to push: specify a bundle directory before registry URL, or use --file to specify tar/chunked package") + } + + Packages = lo.Uniq(Packages) + + if TempDir == "" { + TempDir = filepath.Join(filepath.Dir(Packages[0]), ".tmp", mirror.TmpMirrorFolderName) + } + + return nil +} + +// collectBundlePathPackages resolves the bundle path argument, which may be a +// directory of packages or a single tar/chunked package, into Packages. +func collectBundlePathPackages(arg string) error { + ImagesBundlePath = filepath.Clean(arg) s, err := os.Stat(ImagesBundlePath) if err != nil { @@ -67,13 +110,19 @@ func validateImagesBundlePathArg(args []string) error { } dirEntries = lo.Filter(dirEntries, func(item os.DirEntry, _ int) bool { - ext := filepath.Ext(item.Name()) - return ext == ".tar" || ext == ".chunk" + return isPackageFile(item.Name()) }) if len(dirEntries) == 0 { return errors.New("no packages found in bundle directory") } + for _, entry := range dirEntries { + // Chunk files (.tar.NNNN.chunk) collapse to a single .tar + // package; the pusher reassembles the chunks at push time. + Packages = append(Packages, canonicalPackagePath(filepath.Join(ImagesBundlePath, entry.Name()))) + } + + // Default temp dir lives inside the bundle directory, as before. if TempDir == "" { TempDir = filepath.Join(ImagesBundlePath, ".tmp", mirror.TmpMirrorFolderName) } @@ -81,17 +130,58 @@ func validateImagesBundlePathArg(args []string) error { return nil } - if bundleExtension := filepath.Ext(ImagesBundlePath); bundleExtension == ".tar" || bundleExtension == ".chunk" { - if TempDir == "" { - TempDir = filepath.Join(filepath.Dir(ImagesBundlePath), ".tmp", mirror.TmpMirrorFolderName) - } - + if isPackageFile(ImagesBundlePath) { + Packages = append(Packages, canonicalPackagePath(ImagesBundlePath)) return nil } return fmt.Errorf("invalid images bundle: must be a directory, tar or a chunked package") } +// collectFilesPackages validates and appends the packages passed via --file. +func collectFilesPackages() error { + for _, f := range Files { + path := filepath.Clean(f) + + s, err := os.Stat(path) + if err != nil { + return fmt.Errorf("could not read package %q: %w", path, err) + } + + if s.IsDir() { + return fmt.Errorf("--file entry %q is a directory, expected a tar or chunked package", path) + } + + if !isPackageFile(path) { + return fmt.Errorf("--file entry %q is not a tar or chunked package", path) + } + + Packages = append(Packages, canonicalPackagePath(path)) + } + + return nil +} + +func isPackageFile(name string) bool { + ext := filepath.Ext(name) + return ext == ".tar" || ext == ".chunk" +} + +// canonicalPackagePath maps a chunk file (.tar.NNNN.chunk) to its canonical +// .tar path so the pusher reassembles all chunks instead of reading a single +// chunk as a whole archive. Plain .tar paths are returned unchanged. +func canonicalPackagePath(path string) string { + if idx := strings.Index(path, ".tar.chunk"); idx != -1 { + return path[:idx] + ".tar" + } + + if idx := strings.Index(path, ".tar."); idx != -1 && filepath.Ext(path) == ".chunk" { + return path[:idx] + ".tar" + } + + return path +} + func validateRegistryCredentials() error { if RegistryPassword != "" && RegistryUsername == "" { return errors.New("registry username not specified") @@ -100,8 +190,8 @@ func validateRegistryCredentials() error { return nil } -func parseAndValidateRegistryURLArg(args []string) error { - registry := strings.NewReplacer("http://", "", "https://", "").Replace(args[1]) +func parseAndValidateRegistryURLArg(registryArg string) error { + registry := strings.NewReplacer("http://", "", "https://", "").Replace(registryArg) if registry == "" { return errors.New(" argument is empty") } diff --git a/internal/mirror/push.go b/internal/mirror/push.go index 610ac165..0e628012 100644 --- a/internal/mirror/push.go +++ b/internal/mirror/push.go @@ -46,8 +46,12 @@ const ( // PushServiceOptions contains configuration options for PushService type PushServiceOptions struct { - // BundleDir is the directory containing the bundle to push + // BundleDir is the directory containing the bundle to push. + // Used as a fallback when Packages is empty. BundleDir string + // Packages is an explicit list of tar/chunked package archive paths to push. + // When set, it takes precedence over scanning BundleDir. + Packages []string // WorkingDir is the temporary directory for unpacking bundles WorkingDir string } @@ -155,24 +159,27 @@ func (svc *PushService) Push(ctx context.Context) error { }) } -// unpackAllPackages unpacks all tar packages from bundle directory into unified directory. +// unpackAllPackages unpacks all tar packages into the unified directory. // All packages are unpacked to the same root - the structure inside each tar // should already have the correct paths. +// +// The package archives come from the explicit Packages list when set, otherwise +// they are discovered by scanning BundleDir. func (svc *PushService) unpackAllPackages(ctx context.Context, dirPath string) error { - entries, err := os.ReadDir(svc.options.BundleDir) + packages, err := svc.resolvePackagePaths() if err != nil { - return fmt.Errorf("read bundle dir: %w", err) + return err } - packages := svc.findPackages(entries) if len(packages) == 0 { - return fmt.Errorf("no packages found in bundle directory %q", svc.options.BundleDir) + return fmt.Errorf("no packages found to push") } svc.userLogger.Infof("Found %d packages to unpack", len(packages)) - for _, pkgName := range packages { - if err := svc.unpackPackage(ctx, dirPath, pkgName); err != nil { + for _, pkgPath := range packages { + pkgName := packageNameFromPath(pkgPath) + if err := svc.unpackPackage(ctx, dirPath, pkgPath, pkgName); err != nil { // Log warning but continue with other packages svc.userLogger.Warnf("Failed to unpack %s: %v", pkgName, err) } @@ -181,7 +188,26 @@ func (svc *PushService) unpackAllPackages(ctx context.Context, dirPath string) e return nil } -// findPackages finds all package names (without .tar extension) in the bundle directory. +// resolvePackagePaths returns the list of package archive paths to push. +// It prefers the explicit Packages list; when empty it falls back to scanning +// BundleDir for tar and chunked packages. +func (svc *PushService) resolvePackagePaths() ([]string, error) { + if len(svc.options.Packages) > 0 { + packages := slices.Clone(svc.options.Packages) + slices.Sort(packages) + + return packages, nil + } + + entries, err := os.ReadDir(svc.options.BundleDir) + if err != nil { + return nil, fmt.Errorf("read bundle dir: %w", err) + } + + return svc.findPackages(entries), nil +} + +// findPackages finds all package archive paths in the bundle directory. // It handles both regular .tar files and chunked packages (.tar.chunk000). func (svc *PushService) findPackages(entries []os.DirEntry) []string { packagesSet := make(map[string]struct{}) @@ -206,7 +232,7 @@ func (svc *PushService) findPackages(entries []os.DirEntry) []string { packages := make([]string, 0, len(packagesSet)) for pkg := range packagesSet { - packages = append(packages, pkg) + packages = append(packages, filepath.Join(svc.options.BundleDir, pkg+".tar")) } slices.Sort(packages) @@ -214,10 +240,23 @@ func (svc *PushService) findPackages(entries []os.DirEntry) []string { return packages } -// unpackPackage unpacks a single package to the unified directory. -func (svc *PushService) unpackPackage(ctx context.Context, dirPath, pkgName string) error { +// packageNameFromPath derives the package name (used for legacy module detection +// during unpack) from a package archive path by stripping its directory and the +// .tar or chunked suffix. +func packageNameFromPath(pkgPath string) string { + name := filepath.Base(pkgPath) + + if idx := strings.Index(name, ".tar.chunk"); idx != -1 { + return name[:idx] + } + + return strings.TrimSuffix(name, ".tar") +} + +// unpackPackage unpacks a single package archive into the unified directory. +func (svc *PushService) unpackPackage(ctx context.Context, dirPath, pkgPath, pkgName string) error { return svc.userLogger.Process(fmt.Sprintf("Unpack %s", pkgName), func() error { - pkg, err := svc.openPackage(pkgName) + pkg, err := openPackage(pkgPath) if err != nil { return fmt.Errorf("open package: %w", err) } @@ -232,21 +271,20 @@ func (svc *PushService) unpackPackage(ctx context.Context, dirPath, pkgName stri }) } -// openPackage opens a package file, trying .tar first, then chunked format. -func (svc *PushService) openPackage(pkgName string) (io.ReadCloser, error) { - tarPath := filepath.Join(svc.options.BundleDir, pkgName+".tar") - - pkg, err := os.Open(tarPath) +// openPackage opens a package archive by path, trying the path as a plain .tar +// first, then falling back to the chunked format. +func openPackage(pkgPath string) (io.ReadCloser, error) { + pkg, err := os.Open(pkgPath) if err == nil { return pkg, nil } if !os.IsNotExist(err) { - return nil, fmt.Errorf("open tar package %q: %w", tarPath, err) + return nil, fmt.Errorf("open tar package %q: %w", pkgPath, err) } - // Try chunked format - return chunked.Open(svc.options.BundleDir, pkgName+".tar") + // Try chunked format: chunks live next to the package as .tar.NNNN.chunk + return chunked.Open(filepath.Dir(pkgPath), filepath.Base(pkgPath)) } // pushAllLayouts recursively walks the directory and pushes each OCI layout found.