sourceprep.go

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package sources

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"time"
	"unicode"

	gogit "github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components"
	"github.com/microsoft/azure-linux-dev-tools/internal/global/opctx"
	"github.com/microsoft/azure-linux-dev-tools/internal/projectconfig"
	"github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders"
	"github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff"
	"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms"
	"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils"
	"github.com/samber/lo"
	"go.szostok.io/version"
)

// MacrosFileExtension is the file extension used for azldev-generated macros files.
const MacrosFileExtension = ".azl.macros"

// MacrosFileHeader is the comment header included at the top of generated macros files.
const MacrosFileHeader = `# Macros file automatically generated by azldev.
# Do not edit manually; changes will be overwritten.`

// SourcePreparer is a utility for acquiring and preparing all input files required to build a component.
// This may include the component's spec file, any loose files that must accompany it, any payload source
// archives and patches required by the spec, etc. Preparation may also include any post-processing of these
// inputs to match the component's configuration within the containing project.
type SourcePreparer interface {
	// PrepareSources prepares the input sources for the given component, writing them to the given output directory.
	// After this function completes successfully, the output directory should contain all files required to build
	// the component, with any post-processing applied as necessary. The only remaining dependencies not contained
	// within the output directory will be build-time dependencies on external packages (RPMs), including those
	// relied on to be present implicitly within the build root, or expressed via BuildRequires or DynamicBuildRequires
	// in the component's spec file and any defaults from the macros used to interpret the spec file.
	PrepareSources(ctx context.Context, component components.Component, outputDir string, applyOverlays bool) error

	// DiffSources computes a unified diff showing the changes that overlays apply to a component's sources.
	// The component's sources are fetched once into a subdirectory of baseDir, then copied to a second
	// subdirectory where overlays are applied in-place. The diff between the two subdirectories is returned.
	DiffSources(ctx context.Context, component components.Component, baseDir string) (*dirdiff.DiffResult, error)
}

// PreparerOption is a functional option for configuring a [SourcePreparer].
type PreparerOption func(*sourcePreparerImpl)

// WithGitRepo returns a [PreparerOption] that enables dist-git repository
// creation during source preparation. When set, the upstream .git directory
// is preserved and synthetic commit history is generated on top of it. This
// requires the project configuration to reside inside a git repository.
// Without this option, no dist-git is created and synthetic history is skipped.
func WithGitRepo() PreparerOption {
	return func(p *sourcePreparerImpl) {
		p.withGitRepo = true
	}
}

// Standard implementation of the [SourcePreparer] interface.
type sourcePreparerImpl struct {
	sourceManager sourceproviders.SourceManager
	fs            opctx.FS
	eventListener opctx.EventListener
	dryRunnable   opctx.DryRunnable

	// withGitRepo, when true, enables dist-git creation by preserving the
	// upstream .git directory and generating synthetic commit history.
	withGitRepo bool
}

// NewPreparer creates a new [SourcePreparer] instance. All positional arguments
// are required. Optional behavior can be configured via [PreparerOption] values.
func NewPreparer(
	sourceManager sourceproviders.SourceManager,
	fs opctx.FS,
	eventListener opctx.EventListener,
	dryRunnable opctx.DryRunnable,
	opts ...PreparerOption,
) (SourcePreparer, error) {
	if sourceManager == nil {
		return nil, errors.New("source manager cannot be nil")
	}

	if fs == nil {
		return nil, errors.New("filesystem interface cannot be nil")
	}

	if eventListener == nil {
		return nil, errors.New("event listener interface cannot be nil")
	}

	if dryRunnable == nil {
		return nil, errors.New("dry runnable interface cannot be nil")
	}

	impl := &sourcePreparerImpl{
		sourceManager: sourceManager,
		fs:            fs,
		eventListener: eventListener,
		dryRunnable:   dryRunnable,
	}

	for _, opt := range opts {
		if opt != nil {
			opt(impl)
		}
	}

	return impl, nil
}

// PrepareSources implements the [SourcePreparer] interface.
func (p *sourcePreparerImpl) PrepareSources(
	ctx context.Context, component components.Component, outputDir string, applyOverlays bool,
) error {
	// Use the source manager to fetch source files (archives, patches, etc.)
	err := p.sourceManager.FetchFiles(ctx, component, outputDir)
	if err != nil {
		return fmt.Errorf("failed to fetch source files for component %#q:\n%w",
			component.GetName(), err)
	}

	// Preserve the upstream .git directory only when dist-git creation is
	// requested via --with-git. This is required so that overlay commits can be
	// appended on top of the upstream commit log during synthetic history generation.
	var fetchOpts []sourceproviders.FetchComponentOption
	if applyOverlays && p.withGitRepo {
		fetchOpts = append(fetchOpts, sourceproviders.WithPreserveGitDir())
	}

	// Use the source manager to fetch the component (spec file and sidecar files).
	err = p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...)
	if err != nil {
		return fmt.Errorf("failed to fetch sources for component %#q:\n%w",
			component.GetName(), err)
	}

	if applyOverlays {
		err := p.applyOverlaysToSources(ctx, component, outputDir)
		if err != nil {
			return err
		}

		// Record the changes as synthetic git history when dist-git creation is enabled.
		if p.withGitRepo {
			if err := p.trySyntheticHistory(component, outputDir); err != nil {
				return fmt.Errorf("failed to generate synthetic history for component %#q:\n%w",
					component.GetName(), err)
			}
		}
	}

	return nil
}

// applyOverlaysToSources writes the macros file and then applies all overlays.
func (p *sourcePreparerImpl) applyOverlaysToSources(
	ctx context.Context, component components.Component, outputDir string,
) error {
	// Emit computed macros to a macros file in the output directory.
	// If the build configuration produces no macros, no file is written and
	// macrosFileName will be empty.
	var macrosFileName string

	macrosFilePath, err := p.writeMacrosFile(component, outputDir)
	if err != nil {
		return fmt.Errorf("failed to write macros file for component %#q:\n%w",
			component.GetName(), err)
	}

	if macrosFilePath != "" {
		macrosFileName = filepath.Base(macrosFilePath)
	}

	// Apply all overlays to prepared sources.
	if err := p.applyOverlays(ctx, component, outputDir, macrosFileName); err != nil {
		return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err)
	}

	return nil
}

// applyOverlays applies all overlays (user-defined and system-generated) to the
// component sources. Overlay application is decoupled from git history generation:
// overlays modify the working tree; synthetic history is recorded separately by
// [trySyntheticHistory].
func (p *sourcePreparerImpl) applyOverlays(
	_ context.Context, component components.Component, sourcesDirPath, macrosFileName string,
) error {
	event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName())
	defer event.End()

	// Resolve the spec path once for all overlay operations in this call.
	absSpecPath, err := p.resolveSpecPath(component, sourcesDirPath)
	if err != nil {
		return err
	}

	// Collect all overlays in application order. This ensures every change is
	// captured in the synthetic history, including build configuration changes.
	allOverlays, err := p.collectOverlays(component, macrosFileName)
	if err != nil {
		return fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err)
	}

	if len(allOverlays) == 0 {
		return nil
	}

	// Apply all overlays to the working tree.
	if err := p.applyOverlayList(allOverlays, sourcesDirPath, absSpecPath); err != nil {
		return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err)
	}

	return nil
}

// collectOverlays gathers all overlays for a component into a single ordered slice:
// macros-load first, then user overlays, followed by check-skip and file-header overlays.
func (p *sourcePreparerImpl) collectOverlays(
	component components.Component, macrosFileName string,
) ([]projectconfig.ComponentOverlay, error) {
	config := component.GetConfig()

	var allOverlays []projectconfig.ComponentOverlay

	if macrosFileName != "" {
		macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName)
		if err != nil {
			return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err)
		}

		allOverlays = append(allOverlays, macroOverlays...)
	}

	allOverlays = append(allOverlays, config.Overlays...)
	allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...)
	allOverlays = append(allOverlays, generateFileHeaderOverlay()...)

	return allOverlays, nil
}

// initSourcesRepo initializes a new git repository in sourcesDirPath, stages all files,
// and creates an initial commit. This is used for components that don't have an upstream
// dist-git so that Affects commits can still be layered on top.
func initSourcesRepo(sourcesDirPath string) (*gogit.Repository, error) {
	slog.Info("Initializing git repository for sources", "path", sourcesDirPath)

	repo, err := gogit.PlainInit(sourcesDirPath, false)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize git repository at %#q:\n%w", sourcesDirPath, err)
	}

	worktree, err := repo.Worktree()
	if err != nil {
		return nil, fmt.Errorf("failed to get worktree:\n%w", err)
	}

	if err := worktree.AddWithOptions(&gogit.AddOptions{All: true}); err != nil {
		return nil, fmt.Errorf("failed to stage files:\n%w", err)
	}

	_, err = worktree.Commit("Initial sources", &gogit.CommitOptions{
		Author: &object.Signature{
			Name: "azldev",
			When: time.Unix(0, 0).UTC(),
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to create initial commit:\n%w", err)
	}

	return repo, nil
}

// trySyntheticHistory attempts to create synthetic git commits on top of the
// component's sources directory. If no .git directory exists, one is initialized
// with an initial commit so Affects commits can be layered on uniformly for all
// component types.
//
// Returns a non-nil error if history generation fails.
func (p *sourcePreparerImpl) trySyntheticHistory(
	component components.Component,
	sourcesDirPath string,
) error {
	config := component.GetConfig()

	// Build commit metadata from Affects commits.
	commits, err := buildSyntheticCommits(config, component.GetName())
	if err != nil {
		return fmt.Errorf("failed to build synthetic commits:\n%w", err)
	}

	if len(commits) == 0 {
		slog.Debug("No synthetic commits to create; skipping history generation",
			"component", component.GetName())

		return nil
	}

	// Adjust the Release tag before staging changes. See [tryBumpStaticRelease]
	// for the handling of %autorelease, static integers, and non-standard values.
	if err := p.tryBumpStaticRelease(component, sourcesDirPath, len(commits)); err != nil {
		return fmt.Errorf("failed to apply release bump:\n%w", err)
	}

	// Use os.Stat (not p.fs) because go-git always operates on the real filesystem.
	gitDirPath := filepath.Join(sourcesDirPath, ".git")

	_, statErr := os.Stat(gitDirPath)

	if statErr != nil && !os.IsNotExist(statErr) {
		return fmt.Errorf("failed to check for .git directory at %#q:\n%w", gitDirPath, statErr)
	}

	if os.IsNotExist(statErr) {
		slog.Info("No .git directory in sources; initializing repository",
			"component", component.GetName())

		if _, err := initSourcesRepo(sourcesDirPath); err != nil {
			return fmt.Errorf("failed to initialize sources repository:\n%w", err)
		}
	}

	// Open the git repository where synthetic commits will be recorded.
	sourcesRepo, err := gogit.PlainOpen(sourcesDirPath)
	if err != nil {
		return fmt.Errorf("failed to open sources repository at %#q:\n%w", sourcesDirPath, err)
	}

	if err := CommitSyntheticHistory(sourcesRepo, commits); err != nil {
		return fmt.Errorf("failed to commit synthetic history:\n%w", err)
	}

	return nil
}

// DiffSources implements the [SourcePreparer] interface.
// It fetches the component's sources once, copies them to a second directory, applies overlays
// to the copy, then diffs the two trees. This avoids fetching the sources twice.
func (p *sourcePreparerImpl) DiffSources(
	ctx context.Context, component components.Component, baseDir string,
) (result *dirdiff.DiffResult, err error) {
	event := p.eventListener.StartEvent("Computing overlay diff", "component", component.GetName())
	defer event.End()

	// Create temp dirs for sources prepared without and with overlays.
	originalDir, err := fileutils.MkdirTemp(p.fs, baseDir, "original-")
	if err != nil {
		return nil, fmt.Errorf("failed to create temp dir for original sources:\n%w", err)
	}

	defer fileutils.RemoveAllAndUpdateErrorIfNil(p.fs, originalDir, &err)

	// Prepare sources without applying overlays, to get the original tree.
	if err := p.PrepareSources(ctx, component, originalDir, false /* applyOverlays */); err != nil {
		return nil, err
	}

	// Copy the fetched sources to a separate directory for in-place overlay application.
	// This avoids a second network fetch.
	overlaidDir, err := fileutils.MkdirTemp(p.fs, baseDir, "overlaid-")
	if err != nil {
		return nil, fmt.Errorf("failed to create temp dir for overlaid sources:\n%w", err)
	}

	defer fileutils.RemoveAllAndUpdateErrorIfNil(p.fs, overlaidDir, &err)

	if err := fileutils.CopyDirRecursive(
		p.dryRunnable, p.fs, originalDir, overlaidDir,
		fileutils.CopyDirOptions{CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}},
	); err != nil {
		return nil, fmt.Errorf("failed to copy sources for component %#q:\n%w", component.GetName(), err)
	}

	// Apply overlays in-place to the copied directory only.
	if err := p.applyOverlaysToSources(ctx, component, overlaidDir); err != nil {
		return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err)
	}

	// Diff the original tree against the overlaid tree.
	result, err = dirdiff.DiffDirs(p.fs, originalDir, overlaidDir)
	if err != nil {
		return nil, fmt.Errorf("failed to diff source trees for component %#q:\n%w",
			component.GetName(), err)
	}

	return result, nil
}

// writeMacrosFile writes a macros file containing the resolved macros for a component.
// This includes with/without flags converted to macro format, and any explicit defines.
// If the build configuration produces no macros, no file is written and an empty path is
// returned. Otherwise, the path to the written macros file is returned.
func (p *sourcePreparerImpl) writeMacrosFile(component components.Component, outputDir string) (string, error) {
	contents := GenerateMacrosFileContents(component.GetConfig().Build)
	if contents == "" {
		return "", nil
	}

	macrosFilePath := filepath.Join(outputDir, component.GetName()+MacrosFileExtension)

	err := fileutils.WriteFile(p.fs, macrosFilePath, []byte(contents), fileperms.PublicFile)
	if err != nil {
		return "", fmt.Errorf("failed to write macros file %#q:\n%w", macrosFilePath, err)
	}

	return macrosFilePath, nil
}

// GenerateMacrosFileContents generates the contents of an RPM macros file from the given
// build configuration. The output uses standard RPM macro file format (%name value) and
// includes a header comment identifying the file as auto-generated.
//
// With flags are converted to %_with_FLAG 1 macros.
// Without flags are converted to %_without_FLAG 1 macros.
// Defines are emitted as %name value macros.
//
// All macros are collected into a single map and sorted alphabetically by name for
// deterministic output. If the same macro is defined multiple times (e.g., via both
// a with flag and an explicit define), the later definition wins. The order of processing
// is: with flags, then without flags, then explicit defines, then undefines.
//
// Undefines remove entries from the generated macros map. This only affects macros that
// originate from the merged TOML configuration (with, without, and defines fields); it
// does not undefine arbitrary system-level RPM macros. The primary use case is allowing
// a component-level TOML file to override a distro-level or project-level default by
// removing a macro that would otherwise be emitted into the macros file.
//
// Note: RPM macro values can contain spaces without special escaping; everything
// after the macro name (and separating whitespace) is treated as the macro body.
//
// If no macros remain after processing (empty config, or all macros removed via
// undefines), an empty string is returned to signal that no macros file is needed.
func GenerateMacrosFileContents(buildConfig projectconfig.ComponentBuildConfig) string {
	// Build a unified map of all macros. Later definitions override earlier ones.
	// Processing order: with flags -> without flags -> explicit defines.
	macros := make(map[string]string)

	// Convert 'with' flags to macros: FLAG -> _with_FLAG = 1
	for _, with := range buildConfig.With {
		macros["_with_"+with] = "1"
	}

	// Convert 'without' flags to macros: FLAG -> _without_FLAG = 1
	for _, without := range buildConfig.Without {
		macros["_without_"+without] = "1"
	}

	// Add explicit macro definitions (these override any conflicting with/without flags).
	for name, value := range buildConfig.Defines {
		macros[name] = value
	}

	// Remove any macros listed in undefines. This allows component-level config to
	// override distro-level default macro definitions by removing them entirely.
	for _, undef := range buildConfig.Undefines {
		delete(macros, undef)
	}

	if len(macros) == 0 {
		return ""
	}

	lines := []string{
		MacrosFileHeader,
	}

	// Sort macro names for deterministic output.
	macroNames := lo.Keys(macros)
	slices.Sort(macroNames)

	for _, name := range macroNames {
		lines = append(lines, fmt.Sprintf("%%%s %s", name, macros[name]))
	}

	return strings.Join(lines, "\n") + "\n"
}

func synthesizeMacroLoadOverlays(macrosFileName string) ([]projectconfig.ComponentOverlay, error) {
	// Basic check that the macros file name is valid and doesn't require escaping.
	if strings.ContainsFunc(macrosFileName, func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '.' && r != '-' && r != '_' && r != '+'
	}) {
		return nil, fmt.Errorf(
			"macros file name %#q contains invalid characters; does the component name contain invalid characters?",
			macrosFileName,
		)
	}

	// We inject an overlay to prepend a line to the spec to load the macros file.
	return []projectconfig.ComponentOverlay{
		{
			// Prepend the %{load:...} directive to the spec.
			Type: projectconfig.ComponentOverlayPrependSpecLines,
			Lines: []string{
				"# All Azure Linux specs with overlays include this macro file, irrespective of" +
					" whether new macros have been added.",
				fmt.Sprintf("%%{load:%%{_sourcedir}/%s}", macrosFileName),
				"",
			},
		},
		{
			// Ensure that the macros file is manifested as a source in the spec so that
			// mock and other tools know it needs to be present in the build root.
			// Use InsertSpecTag to place it after the last existing Source* tag, avoiding
			// misplacement after macros like %fontpkg or inside %if conditionals.
			Type:  projectconfig.ComponentOverlayInsertSpecTag,
			Tag:   "Source9999", // Use a high number to avoid conflicts with existing sources.
			Value: macrosFileName,
		},
	}, nil
}

// generateFileHeaderOverlay generates an overlay that prepends a header to the spec.
// It should be applied after all other overlays, as it should be the first thing in the spec file.
func generateFileHeaderOverlay() []projectconfig.ComponentOverlay {
	ver := version.Get()

	return []projectconfig.ComponentOverlay{
		{
			Type: projectconfig.ComponentOverlayPrependSpecLines,
			Lines: []string{
				"# This spec file has been modified by azldev to include build configuration overlays. Version: " + ver.Version,
				"# Do not edit manually; changes may be overwritten.",
				"",
			},
		},
	}
}

// synthesizeCheckSkipOverlays generates overlays to disable the %check section if configured.
// When check.skip is true, it prepends an 'exit 0' to the %check section with a comment
// explaining why the section was disabled.
func synthesizeCheckSkipOverlays(checkConfig projectconfig.CheckConfig) []projectconfig.ComponentOverlay {
	if !checkConfig.Skip {
		return nil
	}

	return []projectconfig.ComponentOverlay{
		{
			Type:        projectconfig.ComponentOverlayPrependSpecLines,
			SectionName: "%check",
			Lines: []string{
				"# Check section disabled: " + checkConfig.SkipReason,
				"exit 0",
				"",
			},
		},
	}
}

// resolveSpecPath locates and returns the absolute path to the component's spec file
// within the given sources directory.
func (p *sourcePreparerImpl) resolveSpecPath(
	component components.Component, sourcesDirPath string,
) (string, error) {
	specPath, err := findSpecInDir(p.fs, component, sourcesDirPath)
	if err != nil {
		return "", fmt.Errorf("failed to find spec in sources dir %#q:\n%w", sourcesDirPath, err)
	}

	absSpecPath, err := filepath.Abs(specPath)
	if err != nil {
		return "", fmt.Errorf("failed to get absolute path for spec %#q:\n%w", specPath, err)
	}

	return absSpecPath, nil
}

// applyOverlayList applies a list of overlays to the component sources sequentially.
func (p *sourcePreparerImpl) applyOverlayList(
	overlays []projectconfig.ComponentOverlay, sourcesDirPath, absSpecPath string,
) error {
	for _, overlay := range overlays {
		if err := ApplyOverlayToSources(
			p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath,
		); err != nil {
			return fmt.Errorf("failed to apply %#q overlay:\n%w", overlay.Type, err)
		}
	}

	return nil
}

func findSpecInDir(
	fs opctx.FS, component components.Component, dirPath string,
) (string, error) {
	specPath := filepath.Join(dirPath, component.GetName()+".spec")

	if _, statErr := fs.Stat(specPath); statErr != nil {
		return specPath, fmt.Errorf("failed to find spec for component %#q in %#q:\n%w",
			component.GetName(), dirPath, statErr,
		)
	}

	return specPath, nil
}