feat(snapshot s3): fingerprint buckets from metadata

cmd/kosli/root.go

@@ -105,6 +105,8 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file,
 
 	// single source of truth for the env type lists shown in flag help texts;
 	// the server is the authority on which types are actually accepted
+	validS3FingerprintSources = "content, metadata"
+
 	validEnvTypesList = "K8S, ECS, S3, lambda, server, docker, azure-apps, cloud-run, logical"
 
 	// single source of truth for the service account privilege list shown in
@@ -248,6 +250,7 @@ The ^.kosli_ignore^ will be treated as part of the artifact like any other file,
 	bucketPathsRegexFlag            = "[optional] The comma separated list of Go regular expressions matched against object keys in the S3 bucket to include when fingerprinting. Cannot be used together with --exclude or --exclude-regex."
 	excludeBucketPathsFlag          = "[optional] The comma separated list of file and/or directory paths in the S3 bucket to exclude when fingerprinting. Paths match by literal prefix. Cannot be used together with --include or --include-regex."
 	excludeBucketPathsRegexFlag     = "[optional] The comma separated list of Go regular expressions matched against object keys in the S3 bucket to exclude when fingerprinting. Cannot be used together with --include or --include-regex."
+	s3FingerprintSourceFlag         = "[defaulted] How to fingerprint the bucket content. Valid sources are: [" + validS3FingerprintSources + "]. 'content' downloads every matching object and hashes it. 'metadata' reads the SHA256 checksum S3 stores for each object instead, which requires every matching object to have been uploaded with a full-object SHA256 checksum. Both produce the same fingerprint and need the same permissions."
 	pathsFlag                       = "The comma separated list of absolute or relative paths of artifact directories or files. Can take glob patterns, but be aware that each matching path will be reported as an artifact."
 	excludePathsFlag                = "[optional] The comma separated list of directories and files to exclude from fingerprinting. Can take glob patterns. Only applicable for --artifact-type dir."
 	serverExcludePathsFlag          = "[optional] The comma separated list of directories and files to exclude from fingerprinting. Can take glob patterns."

cmd/kosli/snapshotS3.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -16,6 +17,16 @@ const snapshotS3LongDesc = snapshotS3ShortDesc + awsAuthDesc + `
 You can report the entire bucket content, or filter some of the content using ^--include^ / ^--exclude^ (literal prefix match) or ^--include-regex^ / ^--exclude-regex^ (Go regular expressions matched against the full object key).
 In all cases, the content is reported as one artifact. If you wish to report separate files/dirs within the same bucket as separate artifacts, you need to run the command twice.
 
+By default the bucket content is fingerprinted by downloading every matching object and hashing it. ^--fingerprint-source metadata^ reads the SHA256 checksum S3 stores for each object instead, which avoids the download, the temporary disk space and the local hashing. Both sources produce the same fingerprint, so a snapshot matches the artifact you attested either way.
+
+Fingerprinting from metadata comes with four conditions:
+- Every matching object must carry a full-object SHA256 checksum. S3 only stores one when the upload asked for it, for example ^aws s3api put-object --checksum-algorithm SHA256^. Objects uploaded without one are reported, and cannot be fingerprinted this way.
+- A multipart upload gets a composite SHA256, which hashes the checksums of the individual parts rather than the object content, so it cannot be used as the object's fingerprint. Such an object can be collapsed into a single part in place with ^aws s3api copy-object --checksum-algorithm SHA256 --copy-source yourBucket/yourKey --bucket yourBucket --key yourKey^.
+- ^.kosli_ignore^ is not applied, because reading it would mean downloading it. A bucket with a ^.kosli_ignore^ at its root is reported rather than fingerprinted without its rules.
+- Object keys must be clean relative paths. S3 allows keys such as ^a//b^ or ^/foo^, which cannot be mapped onto a directory tree unambiguously — ^a//b^ and ^a/b^ would collide. Such keys are reported rather than fingerprinted; the default ^content^ source silently collapses them instead.
+
+It does not reduce the permissions the command needs: AWS requires ^s3:GetObject^ to read an object's checksum, the same permission that downloading it needs. Reading the checksum of an SSE-KMS encrypted object additionally needs ^kms:GenerateDataKey^ and ^kms:Decrypt^.
+
 ` + kosliIgnoreDesc
 
 const snapshotS3Example = `
@@ -66,15 +77,30 @@ kosli snapshot s3 yourEnvironmentName \
 	--exclude-regex '.*\.png$' \
 	--api-token yourAPIToken \
 	--org yourOrgName
+
+# report contents of an AWS S3 bucket without downloading the objects,
+# using the SHA256 checksums S3 stores for them:
+kosli snapshot s3 yourEnvironmentName \
+	--bucket yourBucketName \
+	--fingerprint-source metadata \
+	--api-token yourAPIToken \
+	--org yourOrgName
 `
 
+// fingerprint sources accepted by --fingerprint-source
+const (
+	fingerprintSourceContent  = "content"
+	fingerprintSourceMetadata = "metadata"
+)
+
 type snapshotS3Options struct {
-	bucket         string
-	includePaths   []string
-	includeRegex   []string
-	excludePaths   []string
-	excludeRegex   []string
-	awsStaticCreds *aws.AWSStaticCreds
+	bucket            string
+	includePaths      []string
+	includeRegex      []string
+	excludePaths      []string
+	excludeRegex      []string
+	fingerprintSource string
+	awsStaticCreds    *aws.AWSStaticCreds
 }
 
 func newSnapshotS3Cmd(out io.Writer) *cobra.Command {
@@ -106,6 +132,12 @@ func newSnapshotS3Cmd(out io.Writer) *cobra.Command {
 				}
 			}
 
+			if o.fingerprintSource != fingerprintSourceContent && o.fingerprintSource != fingerprintSourceMetadata {
+				return ErrorBeforePrintingUsage(cmd, fmt.Sprintf(
+					"%s is not a valid fingerprint source. Valid sources are: [%s]",
+					o.fingerprintSource, validS3FingerprintSources))
+			}
+
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -118,6 +150,7 @@ func newSnapshotS3Cmd(out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&o.includeRegex, "include-regex", []string{}, bucketPathsRegexFlag)
 	cmd.Flags().StringSliceVarP(&o.excludePaths, "exclude", "x", []string{}, excludeBucketPathsFlag)
 	cmd.Flags().StringSliceVar(&o.excludeRegex, "exclude-regex", []string{}, excludeBucketPathsRegexFlag)
+	cmd.Flags().StringVar(&o.fingerprintSource, "fingerprint-source", fingerprintSourceContent, s3FingerprintSourceFlag)
 	addAWSAuthFlags(cmd, o.awsStaticCreds)
 	addDryRunFlag(cmd)
 
@@ -141,7 +174,12 @@ func (o *snapshotS3Options) run(args []string) error {
 		return err
 	}
 
-	s3Data, err := o.awsStaticCreds.GetS3Data(o.bucket, o.includePaths, o.includeRegex, o.excludePaths, o.excludeRegex, logger)
+	harvest := o.awsStaticCreds.GetS3Data
+	if o.fingerprintSource == fingerprintSourceMetadata {
+		harvest = o.awsStaticCreds.GetS3DataFromMetadata
+	}
+
+	s3Data, err := harvest(o.bucket, o.includePaths, o.includeRegex, o.excludePaths, o.excludeRegex, logger)
 	if err != nil {
 		return err
 	}

cmd/kosli/snapshotS3_test.go

@@ -1,9 +1,13 @@
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
 	"testing"
 
+	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+
 	"github.com/kosli-dev/cli/internal/aws"
 	"github.com/stretchr/testify/suite"
 )
@@ -33,12 +37,22 @@ func (suite *SnapshotS3TestSuite) SetupTest() {
 	// Inject a fake S3 client so tests run without AWS credentials.
 	// The fake is seeded with the objects the test cases filter on.
 	bucketName := suite.bucketName
+	objects := map[string][]byte{
+		"README.md":                  []byte("# kosli cli public\n"),
+		"dummy/dummy_2/template.yml": []byte("key: value\n"),
+	}
+	// Only README.md carries a stored checksum, so the metadata cases cover both
+	// an object that can be fingerprinted from metadata and one that cannot.
+	readmeSum := sha256.Sum256(objects["README.md"])
 	aws.NewS3ClientFunc = func(_ *aws.AWSStaticCreds) (aws.S3API, error) {
 		return &aws.FakeS3Client{
-			Bucket: bucketName,
-			Objects: map[string][]byte{
-				"README.md":                  []byte("# kosli cli public\n"),
-				"dummy/dummy_2/template.yml": []byte("key: value\n"),
+			Bucket:  bucketName,
+			Objects: objects,
+			Checksums: map[string]aws.FakeS3Checksum{
+				"README.md": {
+					SHA256: base64.StdEncoding.EncodeToString(readmeSum[:]),
+					Type:   s3Types.ChecksumTypeFullObject,
+				},
 			},
 		}, nil
 	}
@@ -113,6 +127,28 @@ func (suite *SnapshotS3TestSuite) TestSnapshotS3Cmd() {
 			cmd:    fmt.Sprintf(`snapshot s3 %s %s --bucket %s --exclude dummy`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
 			golden: "bucket kosli-cli-public was reported to environment snapshot-s3-env\n",
 		},
+		{
+			name:   "--fingerprint-source metadata fingerprints from the stored checksum",
+			cmd:    fmt.Sprintf(`snapshot s3 %s %s --bucket %s --include README.md --fingerprint-source metadata`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden: "bucket kosli-cli-public was reported to environment snapshot-s3-env\n",
+		},
+		{
+			name:   "--fingerprint-source content is the default behaviour",
+			cmd:    fmt.Sprintf(`snapshot s3 %s %s --bucket %s --fingerprint-source content`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden: "bucket kosli-cli-public was reported to environment snapshot-s3-env\n",
+		},
+		{
+			wantError: true,
+			name:      "--fingerprint-source rejects an unknown value",
+			cmd:       fmt.Sprintf(`snapshot s3 %s %s --bucket %s --fingerprint-source etag`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden:    "Error: etag is not a valid fingerprint source. Valid sources are: [content, metadata]\nUsage: kosli snapshot s3 ENVIRONMENT-NAME [flags]\n",
+		},
+		{
+			wantError: true,
+			name:      "--fingerprint-source metadata fails on an object with no stored checksum",
+			cmd:       fmt.Sprintf(`snapshot s3 %s %s --bucket %s --include dummy --fingerprint-source metadata`, suite.envName, suite.defaultKosliArguments, suite.bucketName),
+			golden:    "Error: object \"dummy/dummy_2/template.yml\" in bucket [kosli-cli-public] has no SHA256 checksum, so its fingerprint cannot be derived from S3 metadata. Re-upload it with a checksum: aws s3api put-object --bucket kosli-cli-public --key dummy/dummy_2/template.yml --body <file> --checksum-algorithm SHA256. Or fingerprint by downloading the objects instead\n",
+		},
 	}
 
 	for _, t := range tests {

internal/aws/aws.go

@@ -149,17 +149,37 @@ type S3DownloadAPI interface {
 	DownloadObject(ctx context.Context, params *transfermanager.DownloadObjectInput, optFns ...func(*transfermanager.Options)) (*transfermanager.DownloadObjectOutput, error)
 }
 
+// S3HeadAPI reads an object's metadata without reading the object itself,
+// including the checksum S3 stores for it. The real *s3.Client satisfies this
+// implicitly.
+//
+// The stored checksum is only returned when the request sets ChecksumMode to
+// ChecksumModeEnabled.
+type S3HeadAPI interface {
+	HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
+}
+
 // S3API is the combined S3 surface that GetS3Data depends on.
 type S3API interface {
 	S3ListAPI
 	S3DownloadAPI
+	S3HeadAPI
 }
 
-// s3Client combines the two real AWS clients that back S3API: *s3.Client for
-// listing and *transfermanager.Client for downloading.
+// S3MetadataAPI is the narrower surface needed to fingerprint a bucket from the
+// checksums S3 already stores. It deliberately excludes S3DownloadAPI, so the
+// type signature alone shows that no object content is read.
+type S3MetadataAPI interface {
+	S3ListAPI
+	S3HeadAPI
+}
+
+// s3Client combines the real AWS clients that back S3API: *s3.Client for
+// listing and metadata, and *transfermanager.Client for downloading.
 type s3Client struct {
 	S3ListAPI
 	S3DownloadAPI
+	S3HeadAPI
 }
 
 // defaultNewS3Client creates a real S3 client from credentials.
@@ -168,7 +188,11 @@ func defaultNewS3Client(creds *AWSStaticCreds) (S3API, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &s3Client{S3ListAPI: client, S3DownloadAPI: transfermanager.New(client)}, nil
+	return &s3Client{
+		S3ListAPI:     client,
+		S3DownloadAPI: transfermanager.New(client),
+		S3HeadAPI:     client,
+	}, nil
 }
 
 // NewS3ClientFunc is the factory used by GetS3Data to create an S3API client.
@@ -360,7 +384,7 @@ func processOneLambdaFunc(lastModified, codeSha256, functionName, packageType st
 	lambdaData.Digests = map[string]string{functionName: codeSha256}
 
 	if packageType == "Zip" {
-		lambdaData.Digests[functionName], err = decodeLambdaFingerprint(codeSha256)
+		lambdaData.Digests[functionName], err = decodeBase64Sha256(codeSha256)
 		if err != nil {
 			return lambdaData, err
 		}
@@ -375,8 +399,10 @@ func formatLambdaLastModified(lastModified string) (time.Time, error) {
 	return time.Parse(layout, lastModified)
 }
 
-// decodeLambdaFingerprint decodes a base64 lambda function fingerprint
-func decodeLambdaFingerprint(fingerprint string) (string, error) {
+// decodeBase64Sha256 converts a Base64-encoded SHA256 digest into the hex form
+// Kosli fingerprints use. AWS reports stored digests in Base64: Lambda's
+// CodeSha256 and an S3 object's checksum both arrive this way.
+func decodeBase64Sha256(fingerprint string) (string, error) {
 	sha256base64, err := base64.StdEncoding.DecodeString(fingerprint)
 	if err != nil {
 		return "", err
@@ -492,40 +518,22 @@ func getS3DataFromClient(client S3API, bucket string, includePaths, includeRegex
 		}
 	}()
 
-	params := &s3.ListObjectsV2Input{
-		Bucket: aws.String(bucket),
+	objects, err := listMatchingS3Objects(client, bucket, includePaths, includeRegexCompiled,
+		excludePaths, excludeRegexCompiled)
+	if err != nil {
+		return s3Data, err
 	}
 
 	var lastModifiedTime *time.Time
-	paginator := s3.NewListObjectsV2Paginator(client, params)
-	for paginator.HasMorePages() {
-		objects, err := paginator.NextPage(context.TODO())
-		if err != nil {
+	for _, object := range objects {
+		if err := downloadFileFromBucket(client, tempDirName, object.key, bucket, logger); err != nil {
 			return s3Data, err
 		}
-
-		for _, object := range objects.Contents {
-			if strings.HasSuffix(*object.Key, "/") { // skip folders
-				continue
-			}
-			if shouldExcludePath(*object.Key, includePaths, includeRegexCompiled, excludePaths, excludeRegexCompiled) {
-				continue
-			}
-			err := downloadFileFromBucket(client, tempDirName, *object.Key, bucket, logger)
-			if err != nil {
-				return s3Data, err
-			}
-
-			if lastModifiedTime == nil || object.LastModified.After(*lastModifiedTime) {
-				lastModifiedTime = object.LastModified
-			}
+		if lastModifiedTime == nil || object.lastModified.After(*lastModifiedTime) {
+			lastModifiedTime = &object.lastModified
 		}
 	}
 
-	if lastModifiedTime == nil {
-		return s3Data, fmt.Errorf("no matching file or dirs in bucket: [%s]", bucket)
-	}
-
 	fileSnapshot, artifactPath, err := containsSingleFile(tempDirName)
 	if err != nil {
 		return s3Data, err
@@ -550,6 +558,48 @@ func getS3DataFromClient(client S3API, bucket string, includePaths, includeRegex
 	return s3Data, nil
 }
 
+// s3Object is a bucket object that passed the include/exclude filters.
+type s3Object struct {
+	key          string
+	lastModified time.Time
+}
+
+// listMatchingS3Objects paginates the bucket and returns the objects that pass
+// the filters, skipping the folder markers S3 returns for explicitly created
+// folders. It errors when nothing matches, because a snapshot of nothing cannot
+// be fingerprinted.
+//
+// Both fingerprint modes list through here, so what a filter selects cannot
+// drift between them.
+func listMatchingS3Objects(client S3ListAPI, bucket string, includePaths []string,
+	includeRegex []*regexp.Regexp, excludePaths []string, excludeRegex []*regexp.Regexp) ([]s3Object, error) {
+	matched := []s3Object{}
+
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+	})
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(context.TODO())
+		if err != nil {
+			return nil, err
+		}
+		for _, object := range page.Contents {
+			if strings.HasSuffix(*object.Key, "/") { // skip folders
+				continue
+			}
+			if shouldExcludePath(*object.Key, includePaths, includeRegex, excludePaths, excludeRegex) {
+				continue
+			}
+			matched = append(matched, s3Object{key: *object.Key, lastModified: *object.LastModified})
+		}
+	}
+
+	if len(matched) == 0 {
+		return nil, fmt.Errorf("no matching file or dirs in bucket: [%s]", bucket)
+	}
+	return matched, nil
+}
+
 func downloadFileFromBucket(downloader S3DownloadAPI, dirName, key, bucket string, logger *logger.Logger) error {
 	file, err := utils.CreateFile(filepath.Join(dirName, key))
 	if err != nil {

internal/aws/aws_test.go

@@ -68,9 +68,9 @@ func (suite *AWSTestSuite) TestDecodeLambdaFingerprint() {
 		},
 	} {
 		suite.Run(t.name, func() {
-			got, err := decodeLambdaFingerprint(t.base64Fingerprint)
+			got, err := decodeBase64Sha256(t.base64Fingerprint)
 			require.False(suite.T(), (err != nil) != t.wantErr,
-				"decodeLambdaFingerprint() error = %v, wantErr %v", err, t.wantErr)
+				"decodeBase64Sha256() error = %v, wantErr %v", err, t.wantErr)
 			if !t.wantErr {
 				require.Equal(suite.T(), t.wantFingerprint, got)
 			}